1 /*
2 * Copyright (c) 2021-2024 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "optimizer/ir/inst.h"
17 #include "optimizer/ir/basicblock.h"
18 #include "optimizer/ir/graph.h"
19 #include "liveness_analyzer.h"
20 #include "optimizer/analysis/dominators_tree.h"
21 #include "optimizer/analysis/loop_analyzer.h"
22 #include "optimizer/optimizations/locations_builder.h"
23 #include "optimizer/optimizations/regalloc/reg_type.h"
24
25 namespace ark::compiler {
LivenessAnalyzer(Graph * graph)26 LivenessAnalyzer::LivenessAnalyzer(Graph *graph)
27 : Analysis(graph),
28 allocator_(graph->GetAllocator()),
29 linearBlocks_(graph->GetAllocator()->Adapter()),
30 instLifeNumbers_(graph->GetAllocator()->Adapter()),
31 instLifeIntervals_(graph->GetAllocator()->Adapter()),
32 instsByLifeNumber_(graph->GetAllocator()->Adapter()),
33 blockLiveRanges_(graph->GetAllocator()->Adapter()),
34 blockLiveSets_(graph->GetLocalAllocator()->Adapter()),
35 pendingCatchPhiInputs_(graph->GetAllocator()->Adapter()),
36 physicalGeneralIntervals_(graph->GetAllocator()->Adapter()),
37 physicalVectorIntervals_(graph->GetAllocator()->Adapter()),
38 intervalsForTemps_(graph->GetAllocator()->Adapter()),
39 useTable_(graph->GetAllocator()),
40 hasSafepointDuringCall_(graph->GetRuntime()->HasSafepointDuringCall())
41 {
42 }
43
RunImpl()44 bool LivenessAnalyzer::RunImpl()
45 {
46 if (!RunLocationsBuilder(GetGraph())) {
47 return false;
48 }
49 GetGraph()->RunPass<DominatorsTree>();
50 GetGraph()->RunPass<LoopAnalyzer>();
51 ResetLiveness();
52 BuildBlocksLinearOrder();
53 BuildInstLifeNumbers();
54 BuildInstLifeIntervals();
55 Finalize();
56 if (!pendingCatchPhiInputs_.empty()) {
57 COMPILER_LOG(ERROR, LIVENESS_ANALYZER)
58 << "Graph contains CatchPhi instructions whose inputs were not processed";
59 return false;
60 }
61 std::copy_if(physicalGeneralIntervals_.begin(), physicalGeneralIntervals_.end(),
62 std::back_inserter(instLifeIntervals_), [](auto li) { return li != nullptr; });
63 std::copy_if(physicalVectorIntervals_.begin(), physicalVectorIntervals_.end(),
64 std::back_inserter(instLifeIntervals_), [](auto li) { return li != nullptr; });
65 std::copy(intervalsForTemps_.begin(), intervalsForTemps_.end(), std::back_inserter(instLifeIntervals_));
66 COMPILER_LOG(DEBUG, LIVENESS_ANALYZER) << "Liveness analysis is constructed";
67 return true;
68 }
69
ResetLiveness()70 void LivenessAnalyzer::ResetLiveness()
71 {
72 instLifeNumbers_.clear();
73 instLifeIntervals_.clear();
74 blockLiveSets_.clear();
75 blockLiveRanges_.clear();
76 physicalGeneralIntervals_.clear();
77 physicalVectorIntervals_.clear();
78 intervalsForTemps_.clear();
79 if (GetGraph()->GetArch() != Arch::NONE) {
80 physicalGeneralIntervals_.resize(REGISTERS_NUM);
81 physicalVectorIntervals_.resize(VREGISTERS_NUM);
82 }
83 #ifndef NDEBUG
84 finalized_ = false;
85 #endif
86 }
87
88 /*
89 * Linear blocks order means:
90 * - all dominators of a block are visiting before this block;
91 * - all blocks belonging to the same loop are contiguous;
92 */
BuildBlocksLinearOrder()93 void LivenessAnalyzer::BuildBlocksLinearOrder()
94 {
95 ASSERT_PRINT(GetGraph()->IsAnalysisValid<DominatorsTree>(), "Liveness Analyzer needs valid Dom Tree");
96 auto size = GetGraph()->GetBlocksRPO().size();
97 linearBlocks_.reserve(size);
98 linearBlocks_.clear();
99 marker_ = GetGraph()->NewMarker();
100 ASSERT_PRINT(marker_ != UNDEF_MARKER, "There are no free markers");
101 if (GetGraph()->IsBytecodeOptimizer() && !GetGraph()->GetTryBeginBlocks().empty()) {
102 LinearizeBlocks<true>();
103 } else {
104 LinearizeBlocks<false>();
105 }
106 ASSERT(linearBlocks_.size() == size);
107 GetGraph()->EraseMarker(marker_);
108 ASSERT_PRINT(CheckLinearOrder(), "Linear block order isn't correct");
109
110 blockLiveSets_.resize(GetGraph()->GetVectorBlocks().size());
111 blockLiveRanges_.resize(GetGraph()->GetVectorBlocks().size());
112 }
113
114 /*
115 * Check if all forward edges of loop header were visited to get the resulting block order in RPO.
116 * Predecessors which are not in the same loop with a header - are forward edges.
117 */
AllForwardEdgesVisited(BasicBlock * block)118 bool LivenessAnalyzer::AllForwardEdgesVisited(BasicBlock *block)
119 {
120 if (!block->IsLoopHeader()) {
121 for (auto pred : block->GetPredsBlocks()) {
122 if (!pred->IsMarked(marker_)) {
123 return false;
124 }
125 }
126 } else {
127 // Head of irreducible loop can not dominate other blocks in the loop
128 if (block->GetLoop()->IsIrreducible()) {
129 return true;
130 }
131 // Predecessors which are not dominated - are forward edges,
132 for (auto pred : block->GetPredsBlocks()) {
133 if (!block->IsDominate(pred) && !pred->IsMarked(marker_)) {
134 return false;
135 }
136 }
137 }
138 return true;
139 }
140
141 template <bool USE_PC_ORDER>
LinearizeBlocks()142 void LivenessAnalyzer::LinearizeBlocks()
143 {
144 ArenaList<BasicBlock *> pending {GetGraph()->GetLocalAllocator()->Adapter()};
145 pending.push_back(GetGraph()->GetStartBlock());
146
147 while (!pending.empty()) {
148 auto current = pending.front();
149 pending.pop_front();
150
151 linearBlocks_.push_back(current);
152 current->SetMarker(marker_);
153
154 auto succs = current->GetSuccsBlocks();
155 // Each block is inserted into pending list before all already inserted blocks
156 // from the same loop. To process edges forwarding to a "true" branches successors
157 // should be processed in reversed order.
158 for (auto it = succs.rbegin(); it != succs.rend(); ++it) {
159 auto succ = *it;
160 if (succ->IsMarked(marker_) || !AllForwardEdgesVisited(succ)) {
161 continue;
162 }
163 InsertSuccToPendingList<USE_PC_ORDER>(pending, succ);
164 }
165 }
166 }
167
168 template <bool USE_PC_ORDER>
InsertSuccToPendingList(ArenaList<BasicBlock * > & pending,BasicBlock * succ)169 void LivenessAnalyzer::InsertSuccToPendingList(ArenaList<BasicBlock *> &pending, BasicBlock *succ)
170 {
171 if constexpr (USE_PC_ORDER) { // NOLINT(readability-braces-around-statements)
172 auto pcCompare = [succ](auto block) { return block->GetGuestPc() > succ->GetGuestPc(); };
173 auto insertBefore = std::find_if(pending.begin(), pending.end(), pcCompare);
174 pending.insert(insertBefore, succ);
175 } else {
176 // Insert successor right before the first block not from an inner loop.
177 // Such ordering guarantee that a loop and all it's inner loops will be processed
178 // before following edges leading to outer loop blocks.
179 auto isNotInnerLoop = [succ](auto block) { return !block->GetLoop()->IsInside(succ->GetLoop()); };
180 auto insertBefore = std::find_if(pending.begin(), pending.end(), isNotInnerLoop);
181 pending.insert(insertBefore, succ);
182 }
183 }
184
185 /*
186 * Check linear order correctness, using dominators tree
187 */
CheckLinearOrder()188 bool LivenessAnalyzer::CheckLinearOrder()
189 {
190 ArenaVector<size_t> blockPos(GetGraph()->GetVectorBlocks().size(), 0, GetAllocator()->Adapter());
191 size_t position = 0;
192 for (auto block : linearBlocks_) {
193 blockPos[block->GetId()] = position++;
194 }
195
196 for (auto block : linearBlocks_) {
197 if (block->GetDominator() != nullptr) {
198 ASSERT_PRINT(blockPos[block->GetDominator()->GetId()] < blockPos[block->GetId()],
199 "Each block should be visited after its dominator");
200 }
201 if (!block->IsTryEnd()) {
202 continue;
203 }
204 ASSERT(block->GetTryId() != INVALID_ID);
205 for (auto bb : linearBlocks_) {
206 if (bb->IsTry() && bb->GetTryId() == block->GetTryId()) {
207 ASSERT_PRINT(blockPos[bb->GetId()] < blockPos[block->GetId()],
208 "Each try-block should be visited before its try-end block");
209 }
210 }
211 }
212
213 return true;
214 }
215
216 /*
217 * Set lifetime number and visiting number for each instruction
218 */
BuildInstLifeNumbers()219 void LivenessAnalyzer::BuildInstLifeNumbers()
220 {
221 LifeNumber blockBegin;
222 LifeNumber lifeNumber = 0;
223 LinearNumber linearNumber = 0;
224
225 for (auto block : GetLinearizedBlocks()) {
226 blockBegin = lifeNumber;
227 // set the same number for each phi in the block
228 for (auto phi : block->PhiInsts()) {
229 phi->SetLinearNumber(linearNumber++);
230 SetInstLifeNumber(phi, lifeNumber);
231 CreateLifeIntervals(phi);
232 }
233 // ignore PHI instructions
234 instsByLifeNumber_.push_back(nullptr);
235 // set a unique number for each instruction in the block, differing by 2
236 // for the reason of adding spill/fill instructions
237 for (auto inst : block->Insts()) {
238 inst->SetLinearNumber(linearNumber++);
239 CreateLifeIntervals(inst);
240 if (IsPseudoUserOfMultiOutput(inst)) {
241 // Should be the same life number as pseudo-user, since actually they have the same definition
242 SetInstLifeNumber(inst, lifeNumber);
243 GetInstLifeIntervals(inst)->AddUsePosition(lifeNumber);
244 continue;
245 }
246 lifeNumber += LIFE_NUMBER_GAP;
247 SetInstLifeNumber(inst, lifeNumber);
248 SetUsePositions(inst, lifeNumber);
249 instsByLifeNumber_.push_back(inst);
250
251 if (inst->RequireTmpReg()) {
252 CreateIntervalForTemp(lifeNumber);
253 }
254 }
255 lifeNumber += LIFE_NUMBER_GAP;
256 SetBlockLiveRange(block, {blockBegin, lifeNumber});
257 }
258 }
259
SetUsePositions(Inst * userInst,LifeNumber lifeNumber)260 void LivenessAnalyzer::SetUsePositions(Inst *userInst, LifeNumber lifeNumber)
261 {
262 if (GetGraph()->IsBytecodeOptimizer()) {
263 return;
264 }
265 if (userInst->IsCatchPhi() || userInst->IsSaveState()) {
266 return;
267 }
268 for (size_t i = 0; i < userInst->GetInputsCount(); i++) {
269 auto location = userInst->GetLocation(i);
270 if (!location.IsAnyRegister()) {
271 continue;
272 }
273 auto inputInst = userInst->GetDataFlowInput(i);
274 auto li = GetInstLifeIntervals(inputInst);
275 if (location.IsRegisterValid()) {
276 useTable_.AddUseOnFixedLocation(inputInst, location, lifeNumber);
277 } else if (location.IsUnallocatedRegister()) {
278 li->AddUsePosition(lifeNumber);
279 }
280 }
281
282 // Constant can be defined without register
283 if (userInst->IsConst() && g_options.IsCompilerRematConst()) {
284 return;
285 }
286
287 // If instruction required dst register, set use position in the beginning of the interval
288 auto li = GetInstLifeIntervals(userInst);
289 if (!li->NoDest()) {
290 li->AddUsePosition(lifeNumber);
291 }
292 }
293
294 /*
295 * Get lifetime intervals for each instruction
296 */
BuildInstLifeIntervals()297 void LivenessAnalyzer::BuildInstLifeIntervals()
298 {
299 for (auto it = GetLinearizedBlocks().rbegin(); it != GetLinearizedBlocks().rend(); it++) {
300 auto block = *it;
301 auto liveSet = GetInitInstLiveSet(block);
302 ProcessBlockLiveInstructions(block, liveSet);
303 }
304 }
305
306 /*
307 * The initial set of live instructions in the `block` is the the union of all live instructions at the beginning of
308 * the block's successors.
309 * Also for each phi-instruction of the successors: input corresponding to the `block` is added to the live set.
310 */
GetInitInstLiveSet(BasicBlock * block)311 InstLiveSet *LivenessAnalyzer::GetInitInstLiveSet(BasicBlock *block)
312 {
313 unsigned instructionCount = instLifeIntervals_.size();
314 auto liveSet = GetAllocator()->New<InstLiveSet>(instructionCount, GetAllocator());
315 for (auto succ : block->GetSuccsBlocks()) {
316 // catch-begin is pseudo successor, its live set will be processed for blocks with throwable instructions
317 if (succ->IsCatchBegin()) {
318 continue;
319 }
320 liveSet->Union(GetBlockLiveSet(succ));
321 for (auto phi : succ->PhiInsts()) {
322 auto phiInput = phi->CastToPhi()->GetPhiDataflowInput(block);
323 liveSet->Add(phiInput->GetLinearNumber());
324 }
325 }
326
327 // if basic block contains throwable instruction, all instructions live in the catch-handlers should be live in this
328 // block
329 if (auto inst = block->GetFistThrowableInst(); inst != nullptr) {
330 auto handlers = GetGraph()->GetThrowableInstHandlers(inst);
331 for (auto catchHandler : handlers) {
332 liveSet->Union(GetBlockLiveSet(catchHandler));
333 }
334 }
335 return liveSet;
336 }
337
GetLoopEnd(Loop * loop)338 LifeNumber LivenessAnalyzer::GetLoopEnd(Loop *loop)
339 {
340 LifeNumber loopEnd = 0;
341 // find max LifeNumber of inner loops
342 for (auto inner : loop->GetInnerLoops()) {
343 loopEnd = std::max(loopEnd, GetLoopEnd(inner));
344 }
345 // find max LifeNumber of back_edges
346 for (auto backEdge : loop->GetBackEdges()) {
347 loopEnd = std::max(loopEnd, GetBlockLiveRange(backEdge).GetEnd());
348 }
349 return loopEnd;
350 }
351
352 /*
353 * Append and adjust the lifetime intervals for each instruction live in the block and for their inputs
354 */
ProcessBlockLiveInstructions(BasicBlock * block,InstLiveSet * liveSet)355 void LivenessAnalyzer::ProcessBlockLiveInstructions(BasicBlock *block, InstLiveSet *liveSet)
356 {
357 // For each live instruction set initial life range equals to the block life range
358 for (auto &interval : instLifeIntervals_) {
359 if (liveSet->IsSet(interval->GetInst()->GetLinearNumber())) {
360 interval->AppendRange(GetBlockLiveRange(block));
361 }
362 }
363
364 for (Inst *inst : block->InstsSafeReverse()) {
365 // Shorten instruction lifetime to the position where its defined
366 // and remove from the block's live set
367 auto instLifeNumber = GetInstLifeNumber(inst);
368 auto interval = GetInstLifeIntervals(inst);
369 interval->StartFrom(instLifeNumber);
370 liveSet->Remove(inst->GetLinearNumber());
371 if (inst->IsCatchPhi()) {
372 // catch-phi's liveness should overlap all linked try blocks' livenesses
373 for (auto pred : inst->GetBasicBlock()->GetPredsBlocks()) {
374 instLifeNumber = std::min(instLifeNumber, GetBlockLiveRange(pred).GetBegin());
375 }
376 interval->StartFrom(instLifeNumber);
377 AdjustCatchPhiInputsLifetime(inst);
378 } else {
379 if (inst->GetOpcode() == Opcode::LiveOut) {
380 ProcessOpcodeLiveOut(block, interval, instLifeNumber);
381 }
382 auto currentLiveRange = LiveRange {GetBlockLiveRange(block).GetBegin(), instLifeNumber};
383 AdjustInputsLifetime(inst, currentLiveRange, liveSet);
384 }
385
386 BlockFixedRegisters(inst);
387 }
388
389 // The lifetime interval of phis instructions starts at the beginning of the block
390 for (auto phi : block->PhiInsts()) {
391 liveSet->Remove(phi->GetLinearNumber());
392 }
393
394 // All instructions live at the beginning ot the loop header
395 // must be live for the entire loop
396 if (block->IsLoopHeader()) {
397 LifeNumber loopEndPosition = GetLoopEnd(block->GetLoop());
398
399 for (auto &interval : instLifeIntervals_) {
400 if (liveSet->IsSet(interval->GetInst()->GetLinearNumber())) {
401 interval->AppendGroupRange({GetBlockLiveRange(block).GetBegin(), loopEndPosition});
402 }
403 }
404 }
405 SetBlockLiveSet(block, liveSet);
406 }
407
ProcessOpcodeLiveOut(BasicBlock * block,LifeIntervals * interval,LifeNumber instLifeNumber)408 void LivenessAnalyzer::ProcessOpcodeLiveOut(BasicBlock *block, LifeIntervals *interval, LifeNumber instLifeNumber)
409 {
410 if (block->GetSuccsBlocks().size() == 1 && block->GetSuccessor(0)->IsEndBlock()) {
411 interval->AppendRange({instLifeNumber, GetBlockLiveRange(block).GetEnd()});
412 } else {
413 interval->AppendRange({instLifeNumber, GetBlockLiveRange(GetGraph()->GetEndBlock()).GetBegin()});
414 }
415 }
416
417 /* static */
GetPropagatedLiveRange(Inst * inst,LiveRange liveRange)418 LiveRange LivenessAnalyzer::GetPropagatedLiveRange(Inst *inst, LiveRange liveRange)
419 {
420 /*
421 * Implicit null check encoded as no-op and if the reference to check is null
422 * then SIGSEGV will be raised at the first (closest) user. Regmap generated for
423 * NullCheck's SaveState should be valid at that user so we need to extend
424 * life intervals of SaveState's inputs until NullCheck user.
425 */
426 if (inst->IsNullCheck() && !inst->GetUsers().Empty() && inst->CastToNullCheck()->IsImplicit()) {
427 auto extendUntil = std::numeric_limits<LifeNumber>::max();
428 for (auto &user : inst->GetUsers()) {
429 // Skip the users dominating the NullCheck as their live ranges
430 // start earlier than NullCheck's live range.
431 if (!user.GetInst()->IsDominate(inst)) {
432 auto li = GetInstLifeIntervals(user.GetInst());
433 ASSERT(li != nullptr);
434 extendUntil = std::min<LifeNumber>(extendUntil, li->GetBegin() + 1);
435 }
436 }
437 liveRange.SetEnd(extendUntil);
438 return liveRange;
439 }
440 /*
441 * We need to propagate liveness for instruction with CallRuntime to save registers before call;
442 * Otherwise, we will not be able to restore the value of the virtual registers
443 */
444 if (inst->IsPropagateLiveness()) {
445 liveRange.SetEnd(liveRange.GetEnd() + 1);
446 } else if (inst->GetOpcode() == Opcode::ReturnInlined && inst->CastToReturnInlined()->IsExtendedLiveness()) {
447 /*
448 * [ReturnInlined]
449 * [ReturnInlined]
450 * ...
451 * [Deoptimize/Throw]
452 *
453 * In this case we propagate ReturnInlined inputs liveness up to the end of basic block
454 */
455 liveRange.SetEnd(GetBlockLiveRange(inst->GetBasicBlock()).GetEnd());
456 }
457 return liveRange;
458 }
459
460 /*
461 * Adjust instruction inputs lifetime and add them to the block's live set
462 */
AdjustInputsLifetime(Inst * inst,LiveRange liveRange,InstLiveSet * liveSet)463 void LivenessAnalyzer::AdjustInputsLifetime(Inst *inst, LiveRange liveRange, InstLiveSet *liveSet)
464 {
465 for (auto input : inst->GetInputs()) {
466 auto inputInst = inst->GetDataFlowInput(input.GetInst());
467 liveSet->Add(inputInst->GetLinearNumber());
468 SetInputRange(inst, inputInst, liveRange);
469 }
470
471 // Extend SaveState inputs lifetime to the end of SaveState's lifetime
472 if (inst->RequireState()) {
473 auto saveState = inst->GetSaveState();
474 auto propagatedRange = GetPropagatedLiveRange(inst, liveRange);
475 while (true) {
476 ASSERT(saveState != nullptr);
477 for (auto ssInput : saveState->GetInputs()) {
478 auto inputInst = saveState->GetDataFlowInput(ssInput.GetInst());
479 liveSet->Add(inputInst->GetLinearNumber());
480 GetInstLifeIntervals(inputInst)->AppendRange(propagatedRange);
481 }
482 auto callerInst = saveState->GetCallerInst();
483 if (callerInst == nullptr) {
484 break;
485 }
486 saveState = callerInst->GetSaveState();
487 }
488 }
489
490 // Handle CatchPhi inputs associated with inst
491 auto range = pendingCatchPhiInputs_.equal_range(inst);
492 for (auto it = range.first; it != range.second; ++it) {
493 auto throwableInput = it->second;
494 auto throwableInputInterval = GetInstLifeIntervals(throwableInput);
495 liveSet->Add(throwableInput->GetLinearNumber());
496 throwableInputInterval->AppendRange(liveRange);
497 }
498 pendingCatchPhiInputs_.erase(inst);
499 }
500
501 /*
502 * Increase ref-input liveness in the 'no-async-jit' mode, since GC can be triggered and delete ref during callee-method
503 * compilation
504 */
SetInputRange(const Inst * inst,const Inst * input,LiveRange liveRange) const505 void LivenessAnalyzer::SetInputRange(const Inst *inst, const Inst *input, LiveRange liveRange) const
506 {
507 if (hasSafepointDuringCall_ && inst->IsCall() && DataType::IsReference(input->GetType())) {
508 GetInstLifeIntervals(input)->AppendRange(liveRange.GetBegin(), liveRange.GetEnd() + 1U);
509 } else {
510 GetInstLifeIntervals(input)->AppendRange(liveRange);
511 }
512 }
513
514 /*
515 * CatchPhi does not handle inputs as regular instructions - instead of performing
516 * some action at CatchPhi's definition copy instruction are added before throwable instructions.
517 * Instead of extending input life interval until CatchPhi it is extended until throwable instruction.
518 */
AdjustCatchPhiInputsLifetime(Inst * inst)519 void LivenessAnalyzer::AdjustCatchPhiInputsLifetime(Inst *inst)
520 {
521 auto catchPhi = inst->CastToCatchPhi();
522
523 for (ssize_t inputIdx = catchPhi->GetInputsCount() - 1; inputIdx >= 0; inputIdx--) {
524 auto inputInst = catchPhi->GetDataFlowInput(inputIdx);
525 auto throwableInst = const_cast<Inst *>(catchPhi->GetThrowableInst(inputIdx));
526
527 pendingCatchPhiInputs_.insert({throwableInst, inputInst});
528 }
529 }
530
SetInstLifeNumber(const Inst * inst,LifeNumber number)531 void LivenessAnalyzer::SetInstLifeNumber([[maybe_unused]] const Inst *inst, LifeNumber number)
532 {
533 ASSERT(instLifeNumbers_.size() == inst->GetLinearNumber());
534 instLifeNumbers_.push_back(number);
535 }
536
GetInstLifeNumber(const Inst * inst) const537 LifeNumber LivenessAnalyzer::GetInstLifeNumber(const Inst *inst) const
538 {
539 return instLifeNumbers_[inst->GetLinearNumber()];
540 }
541
542 /*
543 * Create new lifetime intervals for instruction, check that instruction linear number is equal to intervals
544 * position in vector
545 */
CreateLifeIntervals(Inst * inst)546 void LivenessAnalyzer::CreateLifeIntervals(Inst *inst)
547 {
548 ASSERT(!finalized_);
549 ASSERT(inst->GetLinearNumber() == instLifeIntervals_.size());
550 auto interval = GetAllocator()->New<LifeIntervals>(GetAllocator(), inst);
551 instLifeIntervals_.push_back(interval);
552 }
553
CreateIntervalForTemp(LifeNumber ln)554 void LivenessAnalyzer::CreateIntervalForTemp(LifeNumber ln)
555 {
556 ASSERT(!finalized_);
557 auto interval = GetAllocator()->New<LifeIntervals>(GetAllocator());
558 interval->AppendRange({ln - 1, ln});
559 interval->AddUsePosition(ln);
560 // DataType is INT64, since general register is reserved (for 32-bits arch will be converted to INT32)
561 interval->SetType(ConvertRegType(GetGraph(), DataType::INT64));
562 intervalsForTemps_.push_back(interval);
563 }
564
GetInstLifeIntervals(const Inst * inst) const565 LifeIntervals *LivenessAnalyzer::GetInstLifeIntervals(const Inst *inst) const
566 {
567 ASSERT(inst->GetLinearNumber() != INVALID_LINEAR_NUM);
568 return instLifeIntervals_[inst->GetLinearNumber()];
569 }
570
SetBlockLiveRange(BasicBlock * block,LiveRange lifeRange)571 void LivenessAnalyzer::SetBlockLiveRange(BasicBlock *block, LiveRange lifeRange)
572 {
573 blockLiveRanges_[block->GetId()] = lifeRange;
574 }
575
GetLinearizedBlocks() const576 const ArenaVector<BasicBlock *> &LivenessAnalyzer::GetLinearizedBlocks() const
577 {
578 return linearBlocks_;
579 }
580
GetInstByLifeNumber(LifeNumber ln) const581 Inst *LivenessAnalyzer::GetInstByLifeNumber(LifeNumber ln) const
582 {
583 return instsByLifeNumber_[ln / LIFE_NUMBER_GAP];
584 }
585
GetBlockCoversPoint(LifeNumber ln) const586 BasicBlock *LivenessAnalyzer::GetBlockCoversPoint(LifeNumber ln) const
587 {
588 for (auto bb : linearBlocks_) {
589 if (GetBlockLiveRange(bb).Contains(ln)) {
590 return bb;
591 }
592 }
593 return nullptr;
594 }
595
Cleanup()596 void LivenessAnalyzer::Cleanup()
597 {
598 for (auto *interv : instLifeIntervals_) {
599 if (!interv->IsPhysical() && !interv->IsPreassigned()) {
600 interv->ClearLocation();
601 }
602 if (interv->GetSibling() != nullptr) {
603 interv->MergeSibling();
604 }
605 }
606 }
607
Finalize()608 void LivenessAnalyzer::Finalize()
609 {
610 for (auto *interv : instLifeIntervals_) {
611 interv->Finalize();
612 }
613 for (auto *interv : intervalsForTemps_) {
614 interv->Finalize();
615 }
616 #ifndef NDEBUG
617 finalized_ = true;
618 #endif
619 }
620
GetLifeIntervals() const621 const ArenaVector<LifeIntervals *> &LivenessAnalyzer::GetLifeIntervals() const
622 {
623 return instLifeIntervals_;
624 }
625
GetBlockLiveRange(const BasicBlock * block) const626 LiveRange LivenessAnalyzer::GetBlockLiveRange(const BasicBlock *block) const
627 {
628 return blockLiveRanges_[block->GetId()];
629 }
630
SetBlockLiveSet(BasicBlock * block,InstLiveSet * liveSet)631 void LivenessAnalyzer::SetBlockLiveSet(BasicBlock *block, InstLiveSet *liveSet)
632 {
633 blockLiveSets_[block->GetId()] = liveSet;
634 }
635
GetBlockLiveSet(BasicBlock * block) const636 InstLiveSet *LivenessAnalyzer::GetBlockLiveSet(BasicBlock *block) const
637 {
638 return blockLiveSets_[block->GetId()];
639 }
640
GetUseTable() const641 const UseTable &LivenessAnalyzer::GetUseTable() const
642 {
643 return useTable_;
644 }
645
GetTmpRegInterval(const Inst * inst)646 LifeIntervals *LivenessAnalyzer::GetTmpRegInterval(const Inst *inst)
647 {
648 auto instLn = GetInstLifeNumber(inst);
649 auto it = std::find_if(intervalsForTemps_.begin(), intervalsForTemps_.end(),
650 [instLn](auto li) { return li->GetBegin() == instLn - 1; });
651 return it == intervalsForTemps_.end() ? nullptr : *it;
652 }
653
GetPassName() const654 const char *LivenessAnalyzer::GetPassName() const
655 {
656 return "LivenessAnalysis";
657 }
658
GetBlocksCount() const659 size_t LivenessAnalyzer::GetBlocksCount() const
660 {
661 return blockLiveRanges_.size();
662 }
663
GetAllocator()664 ArenaAllocator *LivenessAnalyzer::GetAllocator()
665 {
666 return allocator_;
667 }
668
DumpLifeIntervals(std::ostream & out) const669 void LivenessAnalyzer::DumpLifeIntervals(std::ostream &out) const
670 {
671 for (auto bb : GetGraph()->GetBlocksRPO()) {
672 if (bb->GetId() >= GetBlocksCount()) {
673 continue;
674 }
675 auto blockRange = GetBlockLiveRange(bb);
676 out << "BB " << bb->GetId() << "\t" << blockRange.ToString() << std::endl;
677
678 for (auto inst : bb->AllInsts()) {
679 if (inst->GetLinearNumber() == INVALID_LINEAR_NUM) {
680 continue;
681 }
682 auto interval = GetInstLifeIntervals(inst);
683
684 out << "v" << inst->GetId() << "\t";
685 for (auto sibling = interval; sibling != nullptr; sibling = sibling->GetSibling()) {
686 out << sibling->ToString<false>() << "@ " << sibling->GetLocation().ToString(GetGraph()->GetArch())
687 << "; ";
688 }
689 out << std::endl;
690 }
691 }
692
693 out << "Temps:" << std::endl;
694 for (auto interval : intervalsForTemps_) {
695 out << interval->ToString<false>() << "@ " << interval->GetLocation().ToString(GetGraph()->GetArch())
696 << std::endl;
697 }
698 DumpLocationsUsage(out);
699 }
700
DumpLocationsUsage(std::ostream & out) const701 void LivenessAnalyzer::DumpLocationsUsage(std::ostream &out) const
702 {
703 std::map<Register, std::vector<LifeIntervals *>> regsIntervals;
704 std::map<Register, std::vector<LifeIntervals *>> vregsIntervals;
705 std::map<Register, std::vector<LifeIntervals *>> slotsIntervals;
706 for (auto &interval : instLifeIntervals_) {
707 for (auto sibling = interval; sibling != nullptr; sibling = sibling->GetSibling()) {
708 auto location = sibling->GetLocation();
709 if (location.IsFpRegister()) {
710 ASSERT(DataType::IsFloatType(interval->GetType()));
711 vregsIntervals[location.GetValue()].push_back(sibling);
712 } else if (location.IsRegister()) {
713 regsIntervals[location.GetValue()].push_back(sibling);
714 } else if (location.IsStack()) {
715 slotsIntervals[location.GetValue()].push_back(sibling);
716 }
717 }
718 }
719
720 for (auto intervalsMap : {®sIntervals, &vregsIntervals, &slotsIntervals}) {
721 std::string locSymbol;
722 if (intervalsMap == ®sIntervals) {
723 out << std::endl << "Registers intervals" << std::endl;
724 locSymbol = "r";
725 } else if (intervalsMap == &vregsIntervals) {
726 out << std::endl << "Vector registers intervals" << std::endl;
727 locSymbol = "vr";
728 } else {
729 ASSERT(intervalsMap == &slotsIntervals);
730 out << std::endl << "Stack slots intervals" << std::endl;
731 locSymbol = "s";
732 }
733
734 if (intervalsMap->empty()) {
735 out << "-" << std::endl;
736 continue;
737 }
738 for (auto &[reg, intervals] : *intervalsMap) {
739 std::sort(intervals.begin(), intervals.end(),
740 [](const auto &lhs, const auto &rhs) { return lhs->GetBegin() < rhs->GetBegin(); });
741 out << locSymbol << std::to_string(reg) << ": ";
742 auto delim = "";
743 for (auto &interval : intervals) {
744 out << delim << interval->ToString<false>();
745 delim = "; ";
746 }
747 out << std::endl;
748 }
749 }
750 }
751
BlockFixedRegisters(Inst * inst)752 void LivenessAnalyzer::BlockFixedRegisters(Inst *inst)
753 {
754 if (GetGraph()->IsBytecodeOptimizer() || GetGraph()->GetMode().IsFastPath()) {
755 return;
756 }
757
758 auto blockFrom = GetInstLifeNumber(inst);
759 if (IsCallBlockingRegisters(inst)) {
760 BlockPhysicalRegisters<false>(blockFrom);
761 BlockPhysicalRegisters<true>(blockFrom);
762 }
763 for (auto i = 0U; i < inst->GetInputsCount(); ++i) {
764 BlockFixedLocationRegister(inst->GetLocation(i), blockFrom);
765 }
766 BlockFixedLocationRegister(inst->GetDstLocation(), blockFrom);
767
768 if (inst->IsParameter()) {
769 // Block a register starting from the position preceding entry block start to
770 // correctly handle register blocked by "current" instruction from registers blocked
771 // by other instructions during register allocation.
772 constexpr auto FIRST_AVAILABLE_LIFE_NUMBER = LIFE_NUMBER_GAP - 1;
773 // Registers holding parameters should be blocked starting from the beginning of entry block
774 // to avoid its assignment to parameter instructions in case of high register pressure.
775 // The required interval is blocked using two calls to correctly mark first use position of a
776 // blocked register.
777 BlockFixedLocationRegister(inst->CastToParameter()->GetLocationData().GetSrc(), blockFrom);
778 BlockFixedLocationRegister(inst->CastToParameter()->GetLocationData().GetSrc(), FIRST_AVAILABLE_LIFE_NUMBER,
779 blockFrom - 1, false);
780 // Block second parameter register that contains number of actual arguments in case of
781 // dynamic methods as it is used in parameter's code generation
782 if (GetGraph()->GetMode().IsDynamicMethod() &&
783 GetGraph()->FindParameter(ParameterInst::DYNAMIC_NUM_ARGS) == nullptr) {
784 BlockReg<false>(Target(GetGraph()->GetArch()).GetParamRegId(1), blockFrom, blockFrom + 1U, true);
785 BlockReg<false>(Target(GetGraph()->GetArch()).GetParamRegId(1), FIRST_AVAILABLE_LIFE_NUMBER, blockFrom - 1,
786 false);
787 }
788 }
789 }
790
791 template <bool IS_FP>
BlockPhysicalRegisters(LifeNumber blockFrom)792 void LivenessAnalyzer::BlockPhysicalRegisters(LifeNumber blockFrom)
793 {
794 auto arch = GetGraph()->GetArch();
795 RegMask callerRegs {GetCallerRegsMask(arch, IS_FP)};
796 for (auto reg = GetFirstCallerReg(arch, IS_FP); reg <= GetLastCallerReg(arch, IS_FP); ++reg) {
797 if (callerRegs.test(reg)) {
798 BlockReg<IS_FP>(reg, blockFrom, blockFrom + 1U, true);
799 }
800 }
801 }
802
BlockFixedLocationRegister(Location location,LifeNumber ln)803 void LivenessAnalyzer::BlockFixedLocationRegister(Location location, LifeNumber ln)
804 {
805 BlockFixedLocationRegister(location, ln, ln + 1U, true);
806 }
807
BlockFixedLocationRegister(Location location,LifeNumber blockFrom,LifeNumber blockTo,bool isUse)808 void LivenessAnalyzer::BlockFixedLocationRegister(Location location, LifeNumber blockFrom, LifeNumber blockTo,
809 bool isUse)
810 {
811 if (location.IsRegister() && location.IsRegisterValid()) {
812 BlockReg<false>(location.GetValue(), blockFrom, blockTo, isUse);
813 } else if (location.IsFpRegister() && location.IsRegisterValid()) {
814 BlockReg<true>(location.GetValue(), blockFrom, blockTo, isUse);
815 }
816 }
817
818 template <bool IS_FP>
BlockReg(Register reg,LifeNumber blockFrom,LifeNumber blockTo,bool isUse)819 void LivenessAnalyzer::BlockReg(Register reg, LifeNumber blockFrom, LifeNumber blockTo, bool isUse)
820 {
821 ASSERT(!finalized_);
822 auto &intervals = IS_FP ? physicalVectorIntervals_ : physicalGeneralIntervals_;
823 auto interval = intervals.at(reg);
824 if (interval == nullptr) {
825 interval = GetGraph()->GetAllocator()->New<LifeIntervals>(GetGraph()->GetAllocator());
826 interval->SetPhysicalReg(reg, IS_FP ? DataType::FLOAT64 : DataType::UINT64);
827 intervals.at(reg) = interval;
828 }
829 interval->AppendRange(blockFrom, blockTo);
830 if (isUse) {
831 interval->AddUsePosition(blockFrom);
832 }
833 }
834
IsCallBlockingRegisters(Inst * inst) const835 bool LivenessAnalyzer::IsCallBlockingRegisters(Inst *inst) const
836 {
837 if (inst->IsCall() && !inst->IsLaunchCall() && !static_cast<CallInst *>(inst)->IsInlined()) {
838 return true;
839 }
840 if (inst->IsIntrinsic() && inst->CastToIntrinsic()->IsNativeCall()) {
841 return true;
842 }
843 return false;
844 }
845
SplitAt(LifeNumber ln,ArenaAllocator * alloc)846 LifeIntervals *LifeIntervals::SplitAt(LifeNumber ln, ArenaAllocator *alloc)
847 {
848 ASSERT(!IsPhysical());
849 ASSERT(ln > GetBegin() && ln <= GetEnd());
850 auto splitChild = alloc->New<LifeIntervals>(alloc, GetInst());
851 #ifndef NDEBUG
852 ASSERT(finalized_);
853 splitChild->finalized_ = true;
854 #endif
855 if (sibling_ != nullptr) {
856 splitChild->sibling_ = sibling_;
857 }
858 splitChild->isSplitSibling_ = true;
859
860 sibling_ = splitChild;
861 splitChild->SetType(GetType());
862
863 auto i = liveRanges_.size();
864 while (i > 0 && liveRanges_[i - 1].GetEnd() <= ln) {
865 i--;
866 }
867 if (i > 0) {
868 auto &range = liveRanges_[i - 1];
869 if (range.GetBegin() < ln) {
870 splitChild->AppendRange(range.GetBegin(), ln);
871 range.SetBegin(ln);
872 }
873 }
874 splitChild->liveRanges_.insert(splitChild->liveRanges_.end(), liveRanges_.begin() + i, liveRanges_.end());
875 liveRanges_.erase(liveRanges_.begin() + i, liveRanges_.end());
876 std::swap(liveRanges_, splitChild->liveRanges_);
877
878 // Move use positions to the child
879 auto it = std::lower_bound(usePositions_.begin(), usePositions_.end(), ln);
880 splitChild->usePositions_.insert(splitChild->usePositions_.end(), it, usePositions_.end());
881 usePositions_.erase(it, usePositions_.end());
882
883 return splitChild;
884 }
885
SplitAroundUses(ArenaAllocator * alloc)886 void LifeIntervals::SplitAroundUses(ArenaAllocator *alloc)
887 {
888 if (GetUsePositions().empty()) {
889 return;
890 }
891 auto use = *GetUsePositions().begin();
892 if (use > GetBegin() + 1) {
893 auto split = SplitAt(use - 1, alloc);
894 split->SplitAroundUses(alloc);
895 } else if (use < GetEnd() - 1) {
896 auto split = SplitAt(use + 1, alloc);
897 split->SplitAroundUses(alloc);
898 }
899 }
900
MergeSibling()901 void LifeIntervals::MergeSibling()
902 {
903 ASSERT(sibling_ != nullptr);
904 #ifndef NDEBUG
905 ASSERT(finalized_);
906 ASSERT(sibling_->finalized_);
907 if (!usePositions_.empty() && !sibling_->usePositions_.empty()) {
908 ASSERT(usePositions_.back() <= sibling_->usePositions_.front());
909 }
910 #endif
911 for (auto range : liveRanges_) {
912 sibling_->AppendRange(range);
913 }
914 liveRanges_ = std::move(sibling_->liveRanges_);
915
916 for (auto &usePos : sibling_->usePositions_) {
917 AddUsePosition(usePos);
918 }
919 sibling_ = nullptr;
920 }
921
FindSiblingAt(LifeNumber ln)922 LifeIntervals *LifeIntervals::FindSiblingAt(LifeNumber ln)
923 {
924 ASSERT(!IsSplitSibling());
925 for (auto head = this; head != nullptr; head = head->GetSibling()) {
926 if (head->GetBegin() <= ln && ln <= head->GetEnd()) {
927 return head;
928 }
929 }
930 return nullptr;
931 }
932
Intersects(const LiveRange & range) const933 bool LifeIntervals::Intersects(const LiveRange &range) const
934 {
935 return // the interval starts within the range
936 (range.GetBegin() <= GetBegin() && GetBegin() <= range.GetEnd()) ||
937 // the interval ends within the range
938 (range.GetBegin() <= GetEnd() && GetEnd() <= range.GetEnd()) ||
939 // the range is fully covered by the interval
940 (GetBegin() <= range.GetBegin() && range.GetEnd() <= GetEnd());
941 }
942
GetFirstIntersectionWith(const LifeIntervals * other,LifeNumber searchFrom) const943 LifeNumber LifeIntervals::GetFirstIntersectionWith(const LifeIntervals *other, LifeNumber searchFrom) const
944 {
945 for (auto it = GetRanges().rbegin(); it != GetRanges().rend(); it++) {
946 auto range = *it;
947 if (range.GetEnd() <= searchFrom) {
948 continue;
949 }
950 for (auto otherIt = other->GetRanges().rbegin(); otherIt != other->GetRanges().rend(); otherIt++) {
951 auto otherRange = *otherIt;
952 if (otherRange.GetEnd() <= searchFrom) {
953 continue;
954 }
955 auto rangeBegin = std::max<LifeNumber>(searchFrom, range.GetBegin());
956 auto otherRangeBegin = std::max<LifeNumber>(searchFrom, otherRange.GetBegin());
957 if (rangeBegin <= otherRangeBegin && otherRangeBegin < range.GetEnd()) {
958 // [range]
959 // [other]
960 return otherRangeBegin;
961 // NOLINTNEXTLINE(readability-else-after-return)
962 } else if (rangeBegin > otherRangeBegin && rangeBegin < otherRange.GetEnd()) {
963 // [range]
964 // [other]
965 return rangeBegin;
966 }
967 }
968 }
969 return INVALID_LIFE_NUMBER;
970 }
971
FindRangeCoveringPosition(LifeNumber ln,LiveRange * dst) const972 bool LifeIntervals::FindRangeCoveringPosition(LifeNumber ln, LiveRange *dst) const
973 {
974 for (auto &range : GetRanges()) {
975 if (range.Contains(ln)) {
976 *dst = range;
977 return true;
978 }
979 }
980
981 return false;
982 }
983
GetSpillWeightAt(const LivenessAnalyzer & la,LifeNumber ln)984 static float GetSpillWeightAt(const LivenessAnalyzer &la, LifeNumber ln)
985 {
986 static constexpr float LOOP_MULT = 10.0;
987 auto block = la.GetBlockCoversPoint(ln);
988 return std::pow<float>(LOOP_MULT, block->GetLoop()->GetDepth());
989 }
990
CalcSpillWeight(const LivenessAnalyzer & la,LifeIntervals * interval)991 float CalcSpillWeight(const LivenessAnalyzer &la, LifeIntervals *interval)
992 {
993 if (interval->IsPhysical()) {
994 return std::numeric_limits<float>::max();
995 }
996
997 size_t length = interval->GetEnd() - interval->GetBegin();
998 // Interval can't be splitted
999 if (length <= LIFE_NUMBER_GAP) {
1000 return std::numeric_limits<float>::max();
1001 }
1002
1003 // Constant intervals are the first candidates to spill
1004 if (interval->GetInst()->IsConst()) {
1005 return std::numeric_limits<float>::min();
1006 }
1007
1008 float useWeight = 0;
1009 if (interval->GetInst()->IsPhi()) {
1010 useWeight += GetSpillWeightAt(la, interval->GetBegin());
1011 }
1012
1013 for (auto use : interval->GetUsePositions()) {
1014 if (use == interval->GetBegin()) {
1015 useWeight += GetSpillWeightAt(la, use + 1);
1016 } else {
1017 useWeight += GetSpillWeightAt(la, use - 1);
1018 }
1019 }
1020 return useWeight;
1021 }
1022
1023 } // namespace ark::compiler
1024