• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Copyright (c) 2021-2023 Huawei Device Co., Ltd.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #include "compiler_logger.h"
17 #include "checks_elimination.h"
18 #include "optimizer/analysis/alias_analysis.h"
19 #include "optimizer/analysis/bounds_analysis.h"
20 #include "optimizer/analysis/dominators_tree.h"
21 #include "optimizer/analysis/loop_analyzer.h"
22 #include "optimizer/analysis/object_type_propagation.h"
23 #include "optimizer/ir/graph_visitor.h"
24 #include "optimizer/ir/analysis.h"
25 
26 namespace panda::compiler {
27 
RunImpl()28 bool ChecksElimination::RunImpl()
29 {
30     COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Start ChecksElimination";
31     GetGraph()->RunPass<DominatorsTree>();
32     GetGraph()->RunPass<LoopAnalyzer>();
33     GetGraph()->RunPass<ObjectTypePropagation>();
34 
35     VisitGraph();
36 
37     if (g_options.IsCompilerEnableReplacingChecksOnDeoptimization()) {
38         if (!GetGraph()->IsOsrMode()) {
39             ReplaceBoundsCheckToDeoptimizationBeforeLoop();
40             MoveCheckOutOfLoop();
41         }
42         ReplaceBoundsCheckToDeoptimizationInLoop();
43 
44         ReplaceCheckMustThrowByUnconditionalDeoptimize();
45     }
46     if (IsLoopDeleted() && GetGraph()->IsOsrMode()) {
47         CleanupGraphSaveStateOSR(GetGraph());
48     }
49     COMPILER_LOG(DEBUG, CHECKS_ELIM) << "ChecksElimination " << (IsApplied() ? "is" : "isn't") << " applied";
50     COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Finish ChecksElimination";
51     return isApplied_;
52 }
53 
InvalidateAnalyses()54 void ChecksElimination::InvalidateAnalyses()
55 {
56     GetGraph()->InvalidateAnalysis<DominatorsTree>();
57     // Already "LoopAnalyzer" was ran in "CleanupGraphSaveStateOSR"
58     // in case (IsLoopDeleted() && GetGraph()->IsOsrMode())
59     if (!(IsLoopDeleted() && GetGraph()->IsOsrMode())) {
60         GetGraph()->InvalidateAnalysis<LoopAnalyzer>();
61     }
62     GetGraph()->InvalidateAnalysis<BoundsAnalysis>();
63     GetGraph()->InvalidateAnalysis<AliasAnalysis>();
64 }
65 
66 template <bool CHECK_FULL_DOM>
VisitNullCheck(GraphVisitor * v,Inst * inst)67 void ChecksElimination::VisitNullCheck(GraphVisitor *v, Inst *inst)
68 {
69     COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Start visit NullCheck with id = " << inst->GetId();
70 
71     auto ref = inst->GetInput(0).GetInst();
72     static_cast<ChecksElimination *>(v)->TryRemoveDominatedNullChecks(inst, ref);
73 
74     if (!static_cast<ChecksElimination *>(v)->TryRemoveCheck<Opcode::NullCheck, CHECK_FULL_DOM>(inst)) {
75         COMPILER_LOG(DEBUG, CHECKS_ELIM) << "NullCheck couldn't be deleted";
76         COMPILER_LOG(DEBUG, CHECKS_ELIM) << "NullCheck saved for further replacing on deoptimization";
77         static_cast<ChecksElimination *>(v)->PushNewCheckForMoveOutOfLoop(inst);
78     }
79 }
80 
VisitNegativeCheck(GraphVisitor * v,Inst * inst)81 void ChecksElimination::VisitNegativeCheck(GraphVisitor *v, Inst *inst)
82 {
83     COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Start visit NegativeCheck with id = " << inst->GetId();
84     if (!static_cast<ChecksElimination *>(v)->TryRemoveCheck<Opcode::NegativeCheck>(inst)) {
85         COMPILER_LOG(DEBUG, CHECKS_ELIM) << "NegativeCheck couldn't be deleted";
86         static_cast<ChecksElimination *>(v)->PushNewCheckForMoveOutOfLoop(inst);
87     }
88 }
89 
VisitNotPositiveCheck(GraphVisitor * v,Inst * inst)90 void ChecksElimination::VisitNotPositiveCheck(GraphVisitor *v, Inst *inst)
91 {
92     COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Start visit NotPositiveCheck with id = " << inst->GetId();
93     if (!static_cast<ChecksElimination *>(v)->TryRemoveCheck<Opcode::NotPositiveCheck>(inst)) {
94         COMPILER_LOG(DEBUG, CHECKS_ELIM) << "NotPositiveCheck couldn't be deleted";
95         static_cast<ChecksElimination *>(v)->PushNewCheckForMoveOutOfLoop(inst);
96     }
97 }
98 
VisitZeroCheck(GraphVisitor * v,Inst * inst)99 void ChecksElimination::VisitZeroCheck(GraphVisitor *v, Inst *inst)
100 {
101     COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Start visit ZeroCheck with id = " << inst->GetId();
102     if (!static_cast<ChecksElimination *>(v)->TryRemoveCheck<Opcode::ZeroCheck>(inst)) {
103         COMPILER_LOG(DEBUG, CHECKS_ELIM) << "ZeroCheck couldn't be deleted";
104         static_cast<ChecksElimination *>(v)->PushNewCheckForMoveOutOfLoop(inst);
105     }
106 }
107 
VisitRefTypeCheck(GraphVisitor * v,Inst * inst)108 void ChecksElimination::VisitRefTypeCheck(GraphVisitor *v, Inst *inst)
109 {
110     auto visitor = static_cast<ChecksElimination *>(v);
111     auto storeInst = inst->GetDataFlowInput(inst->GetInput(1).GetInst());
112     // Case: a[i] = nullptr
113     if (storeInst->GetOpcode() == Opcode::NullPtr) {
114         visitor->ReplaceUsersAndRemoveCheck(inst, storeInst);
115         return;
116     }
117     auto arrayInst = inst->GetDataFlowInput(0);
118     auto ref = inst->GetInput(1).GetInst();
119     // Case:
120     // a[1] = obj
121     // a[2] = obj
122     visitor->TryRemoveDominatedChecks<Opcode::RefTypeCheck>(inst, [arrayInst, ref](Inst *userInst) {
123         return userInst->GetDataFlowInput(0) == arrayInst && userInst->GetInput(1) == ref;
124     });
125     visitor->GetGraph()->RunPass<ObjectTypePropagation>();
126     auto arrayTypeInfo = arrayInst->GetObjectTypeInfo();
127     if (arrayTypeInfo && arrayTypeInfo.IsExact()) {
128         auto storeTypeInfo = storeInst->GetObjectTypeInfo();
129         auto arrayClass = arrayTypeInfo.GetClass();
130         auto storeClass = (storeTypeInfo) ? storeTypeInfo.GetClass() : nullptr;
131         if (visitor->GetGraph()->GetRuntime()->CheckStoreArray(arrayClass, storeClass)) {
132             visitor->ReplaceUsersAndRemoveCheck(inst, storeInst);
133         }
134     }
135 }
136 
TryToEliminateAnyTypeCheck(Inst * inst,Inst * instToReplace,AnyBaseType type,AnyBaseType prevType)137 bool ChecksElimination::TryToEliminateAnyTypeCheck(Inst *inst, Inst *instToReplace, AnyBaseType type,
138                                                    AnyBaseType prevType)
139 {
140     auto language = GetGraph()->GetRuntime()->GetMethodSourceLanguage(GetGraph()->GetMethod());
141     auto allowedType = inst->CastToAnyTypeCheck()->GetAllowedInputType();
142     profiling::AnyInputType prevAllowedType;
143     if (instToReplace->GetOpcode() == Opcode::AnyTypeCheck) {
144         prevAllowedType = instToReplace->CastToAnyTypeCheck()->GetAllowedInputType();
145     } else {
146         prevAllowedType = instToReplace->CastToCastValueToAnyType()->GetAllowedInputType();
147     }
148     auto res = IsAnyTypeCanBeSubtypeOf(language, type, prevType, allowedType, prevAllowedType);
149     if (!res) {
150         return false;
151     }
152     if (res.value()) {
153         ReplaceUsersAndRemoveCheck(inst, instToReplace);
154     } else {
155         PushNewCheckMustThrow(inst);
156     }
157     return true;
158 }
159 
UpdateHclassChecks(Inst * inst)160 void ChecksElimination::UpdateHclassChecks(Inst *inst)
161 {
162     // AnyTypeCheck HEAP_OBJECT => LoadObject Class => LoadObject Hclass => HclassCheck
163     bool allUsersInlined = false;
164     for (auto &user : inst->GetUsers()) {
165         auto userInst = user.GetInst();
166         if (userInst->IsCall()) {
167             if (userInst->CastToCallDynamic()->IsInlined()) {
168                 allUsersInlined = true;
169             } else {
170                 return;
171             }
172         } else if (userInst->GetOpcode() == Opcode::LoadObject) {
173             if (userInst->CastToLoadObject()->GetObjectType() == ObjectType::MEM_DYN_CLASS) {
174                 continue;
175             }
176             ASSERT(userInst->CastToLoadObject()->GetObjectType() == ObjectType::MEM_DYN_METHOD);
177             if (!IsInlinedCallLoadMethod(userInst)) {
178                 return;
179             }
180             allUsersInlined = true;
181         } else {
182             return;
183         }
184     }
185     if (!allUsersInlined) {
186         return;
187     }
188     for (auto &user : inst->GetUsers()) {
189         auto userInst = user.GetInst();
190         auto check = GetHclassCheckFromLoads(userInst);
191         if (!check.has_value()) {
192             continue;
193         }
194         check.value()->CastToHclassCheck()->SetCheckFunctionIsNotClassConstructor(false);
195         SetApplied();
196     }
197 }
198 
GetHclassCheckFromLoads(Inst * loadClass)199 std::optional<Inst *> ChecksElimination::GetHclassCheckFromLoads(Inst *loadClass)
200 {
201     if (loadClass->GetOpcode() != Opcode::LoadObject ||
202         loadClass->CastToLoadObject()->GetObjectType() != ObjectType::MEM_DYN_CLASS) {
203         return std::nullopt;
204     }
205     for (auto &usersLoad : loadClass->GetUsers()) {
206         auto loadHclass = usersLoad.GetInst();
207         if (loadHclass->GetOpcode() != Opcode::LoadObject ||
208             loadHclass->CastToLoadObject()->GetObjectType() != ObjectType::MEM_DYN_HCLASS) {
209             continue;
210         }
211         for (auto &usersHclass : loadHclass->GetUsers()) {
212             auto hclassCheck = usersHclass.GetInst();
213             if (hclassCheck->GetOpcode() == Opcode::HclassCheck) {
214                 return hclassCheck;
215             }
216         }
217     }
218     return std::nullopt;
219 }
220 
IsInlinedCallLoadMethod(Inst * inst)221 bool ChecksElimination::IsInlinedCallLoadMethod(Inst *inst)
222 {
223     bool isMethod = inst->GetOpcode() == Opcode::LoadObject &&
224                     inst->CastToLoadObject()->GetObjectType() == ObjectType::MEM_DYN_METHOD;
225     if (!isMethod) {
226         return false;
227     }
228     for (auto &methodUser : inst->GetUsers()) {
229         auto compare = methodUser.GetInst()->CastToCompare();
230         bool isCompare = compare->GetOpcode() == Opcode::Compare && compare->GetCc() == ConditionCode::CC_NE &&
231                          compare->GetInputType(0) == DataType::POINTER;
232         if (!isCompare) {
233             return false;
234         }
235         auto deoptimizeIf = compare->GetFirstUser()->GetInst()->CastToDeoptimizeIf();
236         if (deoptimizeIf->GetDeoptimizeType() != DeoptimizeType::INLINE_DYN) {
237             return false;
238         }
239     }
240     return true;
241 }
242 
TryRemoveDominatedHclassCheck(Inst * inst)243 void ChecksElimination::TryRemoveDominatedHclassCheck(Inst *inst)
244 {
245     ASSERT(inst->GetOpcode() == Opcode::HclassCheck);
246     // AnyTypeCheck HEAP_OBJECT => LoadObject Class => LoadObject Hclass => HclassCheck
247     auto object = inst->GetInput(0).GetInst()->GetInput(0).GetInst()->GetInput(0).GetInst();
248 
249     for (auto &user : object->GetUsers()) {
250         auto userInst = user.GetInst();
251         auto hclassCheck = GetHclassCheckFromLoads(userInst);
252         if (hclassCheck.has_value() && hclassCheck.value() != inst && inst->IsDominate(hclassCheck.value())) {
253             ASSERT(inst->IsDominate(hclassCheck.value()));
254             COMPILER_LOG(DEBUG, CHECKS_ELIM)
255                 << GetOpcodeString(Opcode::HclassCheck) << " with id = " << inst->GetId() << " dominate on "
256                 << GetOpcodeString(Opcode::HclassCheck) << " with id = " << hclassCheck.value()->GetId();
257             auto block = hclassCheck.value()->GetBasicBlock();
258             auto graph = block->GetGraph();
259             hclassCheck.value()->RemoveInputs();
260             // We don't should update because it's possible to delete one flag in check, when all DynCall is inlined
261             inst->CastToHclassCheck()->ExtendFlags(hclassCheck.value());
262             block->ReplaceInst(hclassCheck.value(), graph->CreateInstNOP());
263             SetApplied();
264         }
265     }
266 }
267 
VisitAnyTypeCheck(GraphVisitor * v,Inst * inst)268 void ChecksElimination::VisitAnyTypeCheck(GraphVisitor *v, Inst *inst)
269 {
270     auto visitor = static_cast<ChecksElimination *>(v);
271     auto inputInst = inst->GetInput(0).GetInst();
272     auto type = inst->CastToAnyTypeCheck()->GetAnyType();
273     if (type == AnyBaseType::UNDEFINED_TYPE) {
274         visitor->ReplaceUsersAndRemoveCheck(inst, inputInst);
275         return;
276     }
277     // from:
278     //     2.any  CastValueToAnyType ANY_SUBTYPE v1 -> (v4)
279     //     4.any  AnyTypeCheck ANY_SUBTYPE v2, v3 -> (....)
280     // to:
281     //     2.any  CastValueToAnyType ANY_SUBTYPE v1 -> (...)
282     if (inputInst->GetOpcode() == Opcode::CastValueToAnyType) {
283         visitor->TryToEliminateAnyTypeCheck(inst, inputInst, type, inputInst->CastToCastValueToAnyType()->GetAnyType());
284         return;
285     }
286     // from:
287     //     2.any  AnyTypeCheck ANY_SUBTYPE v1, v0 -> (v4)
288     //     4.any  AnyTypeCheck ANY_SUBTYPE v2, v3 -> (....)
289     // to:
290     //     2.any  AnyTypeCheck ANY_SUBTYPE v1, v0 -> (...)
291     if (inputInst->GetOpcode() == Opcode::AnyTypeCheck) {
292         visitor->TryToEliminateAnyTypeCheck(inst, inputInst, type, inputInst->CastToAnyTypeCheck()->GetAnyType());
293         return;
294     }
295     // from:
296     //     2.any  AnyTypeCheck ANY_SUBTYPE v1, v0 -> (v4)
297     //     4.any  AnyTypeCheck ANY_SUBTYPE v1, v3 -> (....)
298     // to:
299     //     2.any  AnyTypeCheck ANY_SUBTYPE v1, v0 -> (v4,...)
300     bool applied = false;
301     for (auto &user : inputInst->GetUsers()) {
302         auto userInst = user.GetInst();
303         if (userInst == inst) {
304             continue;
305         }
306         if (userInst->GetOpcode() != Opcode::AnyTypeCheck) {
307             continue;
308         }
309         if (!inst->IsDominate(userInst)) {
310             continue;
311         }
312 
313         if (visitor->TryToEliminateAnyTypeCheck(userInst, inst, userInst->CastToAnyTypeCheck()->GetAnyType(), type)) {
314             applied = true;
315         }
316     }
317     if (!applied) {
318         visitor->PushNewCheckForMoveOutOfLoop(inst);
319     }
320     // from:
321     //    39.any  AnyTypeCheck ECMASCRIPT_HEAP_OBJECT_TYPE v37, v38 -> (v65, v40)
322     //    40.ref  LoadObject 4294967288 Class v39 -> (v41)
323     //    41.ref  LoadObject 4294967287 Hclass v40
324     //    42.     HclassCheck [IsFunc, IsNotClassConstr] v41, v38
325     //      ...METHOD IS INLINED ...
326     // to:
327     //    39.any  AnyTypeCheck ECMASCRIPT_HEAP_OBJECT_TYPE v37, v38 -> (v65, v40)
328     //    40.ref  LoadObject 4294967288 Class v39 -> (v41)
329     //    41.ref  LoadObject 4294967287 Hclass v40
330     //    42.     HclassCheck [IsFunc] v41, v38
331     //      ...METHOD IS INLINED ...
332     if (TryIsDynHeapObject(type)) {
333         visitor->UpdateHclassChecks(inst);
334     }
335 }
336 
VisitHclassCheck(GraphVisitor * v,Inst * inst)337 void ChecksElimination::VisitHclassCheck(GraphVisitor *v, Inst *inst)
338 {
339     [[maybe_unused]] auto visitor = static_cast<ChecksElimination *>(v);
340     visitor->TryRemoveDominatedHclassCheck(inst);
341     visitor->PushNewCheckForMoveOutOfLoop(inst);
342 }
343 
VisitBoundsCheck(GraphVisitor * v,Inst * inst)344 void ChecksElimination::VisitBoundsCheck(GraphVisitor *v, Inst *inst)
345 {
346     COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Start visit BoundsCheck with id = " << inst->GetId();
347     auto block = inst->GetBasicBlock();
348     auto lenArray = inst->GetInput(0).GetInst();
349     auto index = inst->GetInput(1).GetInst();
350     auto visitor = static_cast<ChecksElimination *>(v);
351 
352     visitor->TryRemoveDominatedChecks<Opcode::BoundsCheck>(inst, [lenArray, index](Inst *userInst) {
353         return lenArray == userInst->GetInput(0) && index == userInst->GetInput(1);
354     });
355 
356     if (index->GetType() == DataType::UINT64) {
357         return;
358     }
359     auto bri = block->GetGraph()->GetBoundsRangeInfo();
360     auto lenArrayRange = bri->FindBoundsRange(block, lenArray);
361     auto indexRange = bri->FindBoundsRange(block, index);
362     auto correctUpper = indexRange.IsLess(lenArrayRange) || indexRange.IsLess(lenArray);
363     auto correctLower = indexRange.IsNotNegative();
364     if (correctUpper && correctLower) {
365         COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Index of BoundsCheck have correct bounds";
366         visitor->ReplaceUsersAndRemoveCheck(inst, index);
367         return;
368     }
369     if (indexRange.IsNegative() || indexRange.IsMoreOrEqual(lenArrayRange)) {
370         COMPILER_LOG(DEBUG, CHECKS_ELIM)
371             << "BoundsCheck have incorrect bounds, saved for replace by unconditional deoptimize";
372         visitor->PushNewCheckMustThrow(inst);
373         return;
374     }
375 
376     COMPILER_LOG(DEBUG, CHECKS_ELIM) << "BoundsCheck saved for further replacing on deoptimization";
377     auto loop = GetLoopForBoundsCheck(block, lenArray, index);
378     visitor->PushNewBoundsCheck(loop, lenArray, index, inst, !correctUpper, !correctLower);
379 }
380 
VisitCheckCast(GraphVisitor * v,Inst * inst)381 void ChecksElimination::VisitCheckCast(GraphVisitor *v, Inst *inst)
382 {
383     auto visitor = static_cast<ChecksElimination *>(v);
384     visitor->GetGraph()->RunPass<ObjectTypePropagation>();
385     auto result = ObjectTypeCheckElimination::TryEliminateCheckCast(inst);
386     if (result != ObjectTypeCheckElimination::CheckCastEliminateType::INVALID) {
387         visitor->SetApplied();
388         if (result == ObjectTypeCheckElimination::CheckCastEliminateType::MUST_THROW) {
389             visitor->PushNewCheckMustThrow(inst);
390         }
391         return;
392     }
393     if (inst->CastToCheckCast()->GetOmitNullCheck()) {
394         return;
395     }
396     auto block = inst->GetBasicBlock();
397     auto bri = block->GetGraph()->GetBoundsRangeInfo();
398     auto inputRange = bri->FindBoundsRange(block, inst->GetInput(0).GetInst());
399     if (inputRange.IsMore(BoundsRange(0))) {
400         visitor->SetApplied();
401         inst->CastToCheckCast()->SetOmitNullCheck(true);
402     }
403 }
404 
VisitIsInstance(GraphVisitor * v,Inst * inst)405 void ChecksElimination::VisitIsInstance(GraphVisitor *v, Inst *inst)
406 {
407     if (inst->CastToIsInstance()->GetOmitNullCheck()) {
408         return;
409     }
410     auto block = inst->GetBasicBlock();
411     auto bri = block->GetGraph()->GetBoundsRangeInfo();
412     auto inputRange = bri->FindBoundsRange(block, inst->GetInput(0).GetInst());
413     if (inputRange.IsMore(BoundsRange(0))) {
414         auto visitor = static_cast<ChecksElimination *>(v);
415         visitor->SetApplied();
416         inst->CastToIsInstance()->SetOmitNullCheck(true);
417     }
418 }
419 
VisitAddOverflowCheck(GraphVisitor * v,Inst * inst)420 void ChecksElimination::VisitAddOverflowCheck(GraphVisitor *v, Inst *inst)
421 {
422     ASSERT(inst->GetType() == DataType::INT32);
423     auto input1 = inst->GetInput(0).GetInst();
424     auto input2 = inst->GetInput(1).GetInst();
425     auto visitor = static_cast<ChecksElimination *>(v);
426     visitor->TryRemoveDominatedChecks<Opcode::AddOverflowCheck>(inst, [input1, input2](Inst *userInst) {
427         return (userInst->GetInput(0) == input1 && userInst->GetInput(1) == input2) ||
428                (userInst->GetInput(0) == input2 && userInst->GetInput(1) == input1);
429     });
430     visitor->TryOptimizeOverflowCheck<Opcode::AddOverflowCheck>(inst);
431 }
432 
VisitSubOverflowCheck(GraphVisitor * v,Inst * inst)433 void ChecksElimination::VisitSubOverflowCheck(GraphVisitor *v, Inst *inst)
434 {
435     ASSERT(inst->GetType() == DataType::INT32);
436     auto input1 = inst->GetInput(0).GetInst();
437     auto input2 = inst->GetInput(1).GetInst();
438     auto visitor = static_cast<ChecksElimination *>(v);
439     visitor->TryRemoveDominatedChecks<Opcode::SubOverflowCheck>(inst, [input1, input2](Inst *userInst) {
440         return (userInst->GetInput(0) == input1 && userInst->GetInput(1) == input2);
441     });
442     visitor->TryOptimizeOverflowCheck<Opcode::SubOverflowCheck>(inst);
443 }
444 
VisitNegOverflowAndZeroCheck(GraphVisitor * v,Inst * inst)445 void ChecksElimination::VisitNegOverflowAndZeroCheck(GraphVisitor *v, Inst *inst)
446 {
447     ASSERT(inst->GetType() == DataType::INT32);
448     auto input1 = inst->GetInput(0).GetInst();
449     auto visitor = static_cast<ChecksElimination *>(v);
450     visitor->TryRemoveDominatedChecks<Opcode::NegOverflowAndZeroCheck>(
451         inst, [input1](Inst *userInst) { return (userInst->GetInput(0) == input1); });
452     visitor->TryOptimizeOverflowCheck<Opcode::NegOverflowAndZeroCheck>(inst);
453 }
454 
ReplaceUsersAndRemoveCheck(Inst * instDel,Inst * instRep)455 void ChecksElimination::ReplaceUsersAndRemoveCheck(Inst *instDel, Inst *instRep)
456 {
457     auto block = instDel->GetBasicBlock();
458     auto graph = block->GetGraph();
459     if (graph->IsOsrMode() && block->GetLoop() != instRep->GetBasicBlock()->GetLoop()) {
460         COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Check couldn't be deleted, becouse in OSR mode we can't replace "
461                                             "instructions with instructions from another loop";
462         return;
463     }
464     instDel->ReplaceUsers(instRep);
465     instDel->RemoveInputs();
466     block->ReplaceInst(instDel, graph->CreateInstNOP());
467     COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Checks elimination delete " << GetOpcodeString(instDel->GetOpcode())
468                                      << " with id " << instDel->GetId();
469     graph->GetEventWriter().EventChecksElimination(GetOpcodeString(instDel->GetOpcode()), instDel->GetId(),
470                                                    instDel->GetPc());
471     SetApplied();
472 }
473 
IsInstIncOrDec(Inst * inst)474 bool ChecksElimination::IsInstIncOrDec(Inst *inst)
475 {
476     return inst->IsAddSub() && inst->GetInput(1).GetInst()->IsConst();
477 }
478 
GetInc(Inst * inst)479 int64_t ChecksElimination::GetInc(Inst *inst)
480 {
481     ASSERT(IsInstIncOrDec(inst));
482     auto val = static_cast<int64_t>(inst->GetInput(1).GetInst()->CastToConstant()->GetIntValue());
483     if (inst->IsSub()) {
484         val = -val;
485     }
486     return val;
487 }
488 
GetLoopForBoundsCheck(BasicBlock * block,Inst * lenArray,Inst * index)489 Loop *ChecksElimination::GetLoopForBoundsCheck(BasicBlock *block, Inst *lenArray, Inst *index)
490 {
491     auto parentIndex = IsInstIncOrDec(index) ? index->GetInput(0).GetInst() : index;
492     auto indexBlock = parentIndex->GetBasicBlock();
493     ASSERT(indexBlock != nullptr);
494     auto indexLoop = indexBlock->GetLoop();
495     if (auto loopInfo = CountableLoopParser(*indexLoop).Parse()) {
496         auto input = lenArray;
497         if (lenArray->GetOpcode() == Opcode::LenArray) {
498             // new len_array can be inserted
499             input = lenArray->GetDataFlowInput(0);
500         }
501         if (loopInfo->index == parentIndex && input->GetBasicBlock()->IsDominate(indexBlock)) {
502             ASSERT(indexBlock == indexLoop->GetHeader());
503             return indexLoop;
504         }
505     }
506     return block->GetLoop();
507 }
508 
InitItemForNewIndex(GroupedBoundsChecks * place,Inst * index,Inst * inst,bool checkUpper,bool checkLower)509 void ChecksElimination::InitItemForNewIndex(GroupedBoundsChecks *place, Inst *index, Inst *inst, bool checkUpper,
510                                             bool checkLower)
511 {
512     ASSERT(inst->GetOpcode() == Opcode::BoundsCheck);
513     InstVector insts(GetGraph()->GetLocalAllocator()->Adapter());
514     insts.push_back(inst);
515     int64_t val = 0;
516     Inst *parentIndex = index;
517     if (IsInstIncOrDec(index)) {
518         val = GetInc(index);
519         parentIndex = index->GetInput(0).GetInst();
520     } else if (index->IsConst()) {
521         val = static_cast<int64_t>(index->CastToConstant()->GetIntValue());
522         parentIndex = nullptr;
523     }
524     auto maxVal = checkUpper ? val : std::numeric_limits<int64_t>::min();
525     auto minVal = checkLower ? val : std::numeric_limits<int64_t>::max();
526     place->emplace(parentIndex, std::make_tuple(insts, maxVal, minVal));
527 }
528 
PushNewBoundsCheck(Loop * loop,Inst * lenArray,Inst * index,Inst * inst,bool checkUpper,bool checkLower)529 void ChecksElimination::PushNewBoundsCheck(Loop *loop, Inst *lenArray, Inst *index, Inst *inst, bool checkUpper,
530                                            bool checkLower)
531 {
532     ASSERT(loop != nullptr && lenArray != nullptr && index != nullptr && inst != nullptr);
533     ASSERT(inst->GetOpcode() == Opcode::BoundsCheck);
534     if (boundsChecks_.find(loop) == boundsChecks_.end()) {
535         auto it1 = boundsChecks_.emplace(loop, GetGraph()->GetLocalAllocator()->Adapter());
536         ASSERT(it1.second);
537         auto it2 = it1.first->second.emplace(lenArray, GetGraph()->GetLocalAllocator()->Adapter());
538         ASSERT(it2.second);
539         InitItemForNewIndex(&it2.first->second, index, inst, checkUpper, checkLower);
540     } else if (boundsChecks_.at(loop).find(lenArray) == boundsChecks_.at(loop).end()) {
541         auto it1 = boundsChecks_.at(loop).emplace(lenArray, GetGraph()->GetLocalAllocator()->Adapter());
542         ASSERT(it1.second);
543         InitItemForNewIndex(&it1.first->second, index, inst, checkUpper, checkLower);
544     } else if (auto &lenABc = boundsChecks_.at(loop).at(lenArray); lenABc.find(index) == lenABc.end()) {
545         auto parentIndex = index;
546         int64_t val {};
547         if (IsInstIncOrDec(index)) {
548             parentIndex = index->GetInput(0).GetInst();
549             val = GetInc(index);
550         } else if (index->IsConst()) {
551             parentIndex = nullptr;
552             val = static_cast<int64_t>(index->CastToConstant()->GetIntValue());
553         }
554         if (parentIndex == index || lenABc.find(parentIndex) == lenABc.end()) {
555             InitItemForNewIndex(&lenABc, index, inst, checkUpper, checkLower);
556         } else {
557             auto &item = lenABc.at(parentIndex);
558             std::get<0>(item).push_back(inst);
559             if (val > std::get<1>(item) && checkUpper) {
560                 std::get<1>(item) = val;
561             } else if (val < std::get<2U>(item) && checkLower) {
562                 std::get<2U>(item) = val;
563             }
564         }
565     } else {
566         auto &item = boundsChecks_.at(loop).at(lenArray).at(index);
567         std::get<0>(item).push_back(inst);
568         if (std::get<1>(item) < 0 && checkUpper) {
569             std::get<1>(item) = 0;
570         }
571         if (std::get<2U>(item) > 0 && checkLower) {
572             std::get<2U>(item) = 0;
573         }
574     }
575 }
576 
TryRemoveDominatedNullChecks(Inst * inst,Inst * ref)577 void ChecksElimination::TryRemoveDominatedNullChecks(Inst *inst, Inst *ref)
578 {
579     for (auto &user : ref->GetUsers()) {
580         auto userInst = user.GetInst();
581         if (((userInst->GetOpcode() == Opcode::IsInstance && !userInst->CastToIsInstance()->GetOmitNullCheck()) ||
582              (userInst->GetOpcode() == Opcode::CheckCast && !userInst->CastToCheckCast()->GetOmitNullCheck())) &&
583             inst->IsDominate(userInst)) {
584             COMPILER_LOG(DEBUG, CHECKS_ELIM)
585                 << "NullCheck with id = " << inst->GetId() << " dominate on " << GetOpcodeString(userInst->GetOpcode())
586                 << " with id = " << userInst->GetId();
587             if (userInst->GetOpcode() == Opcode::IsInstance) {
588                 userInst->CastToIsInstance()->SetOmitNullCheck(true);
589             } else {
590                 userInst->CastToCheckCast()->SetOmitNullCheck(true);
591             }
592             SetApplied();
593         }
594     }
595 }
596 
597 template <Opcode OPC, bool CHECK_FULL_DOM, typename CheckInputs>
TryRemoveDominatedChecks(Inst * inst,CheckInputs checkInputs)598 void ChecksElimination::TryRemoveDominatedChecks(Inst *inst, CheckInputs checkInputs)
599 {
600     for (auto &user : inst->GetInput(0).GetInst()->GetUsers()) {
601         auto userInst = user.GetInst();
602         // NOLINTNEXTLINE(readability-magic-numbers)
603         if (userInst->GetOpcode() == OPC && userInst != inst && userInst->GetType() == inst->GetType() &&
604             checkInputs(userInst) &&
605             (CHECK_FULL_DOM ? inst->IsDominate(userInst) : inst->InSameBlockOrDominate(userInst))) {
606             ASSERT(inst->IsDominate(userInst));
607             COMPILER_LOG(DEBUG, CHECKS_ELIM)
608                 // NOLINTNEXTLINE(readability-magic-numbers)
609                 << GetOpcodeString(OPC) << " with id = " << inst->GetId() << " dominate on " << GetOpcodeString(OPC)
610                 << " with id = " << userInst->GetId();
611             ReplaceUsersAndRemoveCheck(userInst, inst);
612         }
613     }
614 }
615 
616 // Remove consecutive checks: NullCheck -> NullCheck -> NullCheck
617 template <Opcode OPC>
TryRemoveConsecutiveChecks(Inst * inst)618 void ChecksElimination::TryRemoveConsecutiveChecks(Inst *inst)
619 {
620     auto end = inst->GetUsers().end();
621     for (auto user = inst->GetUsers().begin(); user != end;) {
622         auto userInst = (*user).GetInst();
623         // NOLINTNEXTLINE(readability-magic-numbers)
624         if (userInst->GetOpcode() == OPC) {
625             // NOLINTNEXTLINE(readability-magic-numbers)
626             COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Remove consecutive " << GetOpcodeString(OPC);
627             ReplaceUsersAndRemoveCheck(userInst, inst);
628             // Start iteration from beginning, because the new successors may be added.
629             user = inst->GetUsers().begin();
630             end = inst->GetUsers().end();
631         } else {
632             ++user;
633         }
634     }
635 }
636 
637 template <Opcode OPC>
TryRemoveCheckByBounds(Inst * inst,Inst * input)638 bool ChecksElimination::TryRemoveCheckByBounds(Inst *inst, Inst *input)
639 {
640     // NOLINTNEXTLINE(readability-magic-numbers)
641     static_assert(OPC == Opcode::ZeroCheck || OPC == Opcode::NegativeCheck || OPC == Opcode::NotPositiveCheck ||
642                   OPC == Opcode::NullCheck);
643     ASSERT(inst->GetOpcode() == OPC);
644     if (input->GetType() == DataType::UINT64) {
645         return false;
646     }
647 
648     auto block = inst->GetBasicBlock();
649     auto bri = block->GetGraph()->GetBoundsRangeInfo();
650     auto range = bri->FindBoundsRange(block, input);
651     bool result = false;
652     // NOLINTNEXTLINE(readability-magic-numbers, readability-braces-around-statements, bugprone-branch-clone)
653     if constexpr (OPC == Opcode::ZeroCheck) {
654         result = range.IsLess(BoundsRange(0)) || range.IsMore(BoundsRange(0));
655     } else if constexpr (OPC == Opcode::NullCheck) {  // NOLINT
656         result = range.IsMore(BoundsRange(0));
657     } else if constexpr (OPC == Opcode::NegativeCheck) {  // NOLINT
658         result = range.IsNotNegative();
659     } else if constexpr (OPC == Opcode::NotPositiveCheck) {  // NOLINT
660         result = range.IsPositive();
661     }
662     if (result) {
663         // NOLINTNEXTLINE(readability-magic-numbers)
664         COMPILER_LOG(DEBUG, CHECKS_ELIM) << GetOpcodeString(OPC) << " have correct bounds";
665         ReplaceUsersAndRemoveCheck(inst, input);
666     } else {
667         // NOLINTNEXTLINE(readability-magic-numbers, readability-braces-around-statements)
668         if constexpr (OPC == Opcode::ZeroCheck || OPC == Opcode::NullCheck) {
669             result = range.IsEqual(BoundsRange(0));
670         } else if constexpr (OPC == Opcode::NegativeCheck) {  // NOLINT
671             result = range.IsNegative();
672         } else if constexpr (OPC == Opcode::NotPositiveCheck) {  // NOLINT
673             result = range.IsNotPositive();
674         }
675         if (result) {
676             COMPILER_LOG(DEBUG, CHECKS_ELIM)
677                 // NOLINTNEXTLINE(readability-magic-numbers)
678                 << GetOpcodeString(OPC) << " must throw, saved for replace by unconditional deoptimize";
679             PushNewCheckMustThrow(inst);
680         }
681     }
682     return result;
683 }
684 
685 template <Opcode OPC, bool CHECK_FULL_DOM>
TryRemoveCheck(Inst * inst)686 bool ChecksElimination::TryRemoveCheck(Inst *inst)
687 {
688     // NOLINTNEXTLINE(readability-magic-numbers)
689     static_assert(OPC == Opcode::ZeroCheck || OPC == Opcode::NegativeCheck || OPC == Opcode::NotPositiveCheck ||
690                   OPC == Opcode::NullCheck);
691     ASSERT(inst->GetOpcode() == OPC);
692 
693     // NOLINTNEXTLINE(readability-magic-numbers)
694     TryRemoveDominatedChecks<OPC, CHECK_FULL_DOM>(inst);
695     // NOLINTNEXTLINE(readability-magic-numbers)
696     TryRemoveConsecutiveChecks<OPC>(inst);
697 
698     auto input = inst->GetInput(0).GetInst();
699     // NOLINTNEXTLINE(readability-magic-numbers)
700     return TryRemoveCheckByBounds<OPC>(inst, input);
701 }
702 
703 template <Opcode OPC>
TryOptimizeOverflowCheck(Inst * inst)704 void ChecksElimination::TryOptimizeOverflowCheck(Inst *inst)
705 {
706     auto block = inst->GetBasicBlock();
707     auto bri = block->GetGraph()->GetBoundsRangeInfo();
708     auto range = bri->FindBoundsRange(block, inst);
709     bool canOverflow = true;
710     if constexpr (OPC == Opcode::NegOverflowAndZeroCheck) {
711         canOverflow = range.CanOverflowNeg(DataType::INT32);
712     } else {
713         canOverflow = range.CanOverflow(DataType::INT32);
714     }
715     if (!canOverflow) {
716         block->RemoveOverflowCheck(inst);
717         SetApplied();
718         return;
719     }
720     bool constInputs = true;
721     for (size_t i = 0; i < inst->GetInputsCount() - 1; ++i) {
722         constInputs &= inst->GetInput(i).GetInst()->IsConst();
723     }
724     if (constInputs) {
725         // replace by deopt
726         PushNewCheckMustThrow(inst);
727         return;
728     }
729     PushNewCheckForMoveOutOfLoop(inst);
730 }
731 
FindSaveState(Loop * loop)732 Inst *ChecksElimination::FindSaveState(Loop *loop)
733 {
734     auto block = loop->GetPreHeader();
735     while (block != nullptr) {
736         for (const auto &inst : block->InstsSafeReverse()) {
737             if (inst->GetOpcode() == Opcode::SaveStateDeoptimize || inst->GetOpcode() == Opcode::SaveState) {
738                 return inst;
739             }
740         }
741         auto next = block->GetDominator();
742         // The case when the dominant block is the head of a inner loop
743         if (next != nullptr && next->GetLoop()->GetOuterLoop() == block->GetLoop()) {
744             return nullptr;
745         }
746         block = next;
747     }
748     return nullptr;
749 }
750 
FindOptimalSaveStateForHoist(Inst * inst,Inst ** optimalInsertAfter)751 Inst *ChecksElimination::FindOptimalSaveStateForHoist(Inst *inst, Inst **optimalInsertAfter)
752 {
753     ASSERT(inst->RequireState());
754     auto block = inst->GetBasicBlock();
755     if (block == nullptr) {
756         return nullptr;
757     }
758     auto loop = block->GetLoop();
759     *optimalInsertAfter = nullptr;
760 
761     while (!loop->IsRoot() && !loop->GetHeader()->IsOsrEntry() && !loop->IsIrreducible()) {
762         for (auto backEdge : loop->GetBackEdges()) {
763             if (!block->IsDominate(backEdge)) {
764                 // avoid taking checks out of slowpath
765                 return *optimalInsertAfter;
766             }
767         }
768         // Find save state
769         Inst *ss = FindSaveState(loop);
770         if (ss == nullptr) {
771             return *optimalInsertAfter;
772         }
773         auto insertAfter = ss;
774 
775         // Check that inputs are dominate on ss
776         bool inputsAreDominate = true;
777         for (size_t i = 0; i < inst->GetInputsCount() - 1; ++i) {
778             auto input = inst->GetInput(i).GetInst();
779             if (input->IsDominate(insertAfter)) {
780                 continue;
781             }
782             if (insertAfter->GetBasicBlock() == input->GetBasicBlock()) {
783                 insertAfter = input;
784             } else {
785                 inputsAreDominate = false;
786                 break;
787             }
788         }
789 
790         if (!inputsAreDominate) {
791             return *optimalInsertAfter;
792         }
793         *optimalInsertAfter = insertAfter;
794         if (insertAfter != ss) {
795             // some inputs are dominate on insert_after but not dominate on ss, stop here
796             // the only case when return value is not equal to *optimal_insert_after
797             return ss;
798         }
799         block = loop->GetHeader();  // block will be used to check for hot path
800         loop = loop->GetOuterLoop();
801     }
802     return *optimalInsertAfter;
803 }
804 
InsertInstAfter(Inst * inst,Inst * after,BasicBlock * block)805 void ChecksElimination::InsertInstAfter(Inst *inst, Inst *after, BasicBlock *block)
806 {
807     if (after->IsPhi()) {
808         block->PrependInst(inst);
809     } else {
810         block->InsertAfter(inst, after);
811     }
812 }
813 
InsertBoundsCheckDeoptimization(ConditionCode cc,Inst * left,int64_t val,Inst * right,Inst * ss,Inst * insertAfter,Opcode newLeftOpcode)814 void ChecksElimination::InsertBoundsCheckDeoptimization(ConditionCode cc, Inst *left, int64_t val, Inst *right,
815                                                         Inst *ss, Inst *insertAfter, Opcode newLeftOpcode)
816 {
817     auto block = insertAfter->GetBasicBlock();
818     Inst *newLeft = nullptr;
819     if (val == 0) {
820         newLeft = left;
821     } else if (left == nullptr) {
822         ASSERT(newLeftOpcode == Opcode::Add);
823         newLeft = GetGraph()->FindOrCreateConstant(val);
824     } else {
825         auto cnst = GetGraph()->FindOrCreateConstant(val);
826         newLeft = GetGraph()->CreateInst(newLeftOpcode);
827         newLeft->SetType(DataType::INT32);
828         newLeft->SetInput(0, left);
829         newLeft->SetInput(1, cnst);
830         if (newLeft->RequireState()) {
831             newLeft->SetSaveState(ss);
832         }
833         InsertInstAfter(newLeft, insertAfter, block);
834         insertAfter = newLeft;
835     }
836     auto deoptComp = GetGraph()->CreateInstCompare(DataType::BOOL, INVALID_PC, newLeft, right, DataType::INT32, cc);
837     auto deopt =
838         GetGraph()->CreateInstDeoptimizeIf(DataType::NO_TYPE, ss->GetPc(), deoptComp, ss, DeoptimizeType::BOUNDS_CHECK);
839     InsertInstAfter(deoptComp, insertAfter, block);
840     block->InsertAfter(deopt, deoptComp);
841 }
842 
InsertDeoptimization(ConditionCode cc,Inst * left,Inst * right,Inst * ss,Inst * insertAfter,DeoptimizeType deoptType)843 Inst *ChecksElimination::InsertDeoptimization(ConditionCode cc, Inst *left, Inst *right, Inst *ss, Inst *insertAfter,
844                                               DeoptimizeType deoptType)
845 {
846     auto deoptComp = GetGraph()->CreateInstCompare(DataType::BOOL, INVALID_PC, left, right, left->GetType(), cc);
847     auto deopt = GetGraph()->CreateInstDeoptimizeIf(DataType::NO_TYPE, ss->GetPc(), deoptComp, ss, deoptType);
848     auto block = insertAfter->GetBasicBlock();
849     block->InsertAfter(deoptComp, insertAfter);
850     block->InsertAfter(deopt, deoptComp);
851     return deopt;
852 }
853 
FindLoopInfo(Loop * loop)854 std::optional<LoopInfo> ChecksElimination::FindLoopInfo(Loop *loop)
855 {
856     Inst *ss = FindSaveState(loop);
857     if (ss == nullptr) {
858         COMPILER_LOG(DEBUG, CHECKS_ELIM) << "SaveState isn't founded";
859         return std::nullopt;
860     }
861     auto loopParser = CountableLoopParser(*loop);
862     if (auto loopInfo = loopParser.Parse()) {
863         auto loopInfoValue = loopInfo.value();
864         if (loopInfoValue.normalizedCc == CC_NE) {
865             return std::nullopt;
866         }
867         bool isHeadLoopExit = loopInfoValue.ifImm->GetBasicBlock() == loop->GetHeader();
868         bool hasPreHeaderCompare = CountableLoopParser::HasPreHeaderCompare(loop, loopInfoValue);
869         ASSERT(loopInfoValue.index->GetOpcode() == Opcode::Phi);
870         if (loopInfoValue.isInc) {
871             return std::make_tuple(loopInfoValue, ss, loopInfoValue.init, loopInfoValue.test,
872                                    loopInfoValue.normalizedCc == CC_LE ? CC_LE : CC_LT, isHeadLoopExit,
873                                    hasPreHeaderCompare);
874         }
875         return std::make_tuple(loopInfoValue, ss, loopInfoValue.test, loopInfoValue.init, CC_LE, isHeadLoopExit,
876                                hasPreHeaderCompare);
877     }
878     COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Not countable loop isn't supported";
879     return std::nullopt;
880 }
881 
InsertNewLenArray(Inst * lenArray,Inst * ss)882 Inst *ChecksElimination::InsertNewLenArray(Inst *lenArray, Inst *ss)
883 {
884     if (lenArray->IsDominate(ss)) {
885         return lenArray;
886     }
887     if (lenArray->GetOpcode() == Opcode::LenArray) {
888         auto nullCheck = lenArray->GetInput(0).GetInst();
889         auto ref = lenArray->GetDataFlowInput(nullCheck);
890         if (ref->IsDominate(ss)) {
891             // Build nullcheck + lenarray before loop
892             auto nullcheck = GetGraph()->CreateInstNullCheck(DataType::REFERENCE, ss->GetPc(), ref, ss);
893             nullcheck->SetFlag(inst_flags::CAN_DEOPTIMIZE);
894             auto newLenArray = lenArray->Clone(GetGraph());
895             newLenArray->SetInput(0, nullcheck);
896             auto block = ss->GetBasicBlock();
897             block->InsertAfter(newLenArray, ss);
898             block->InsertAfter(nullcheck, ss);
899             COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Builded new NullCheck(id=" << nullcheck->GetId()
900                                              << ") and LenArray(id=" << newLenArray->GetId() << ") before loop";
901             ChecksElimination::VisitNullCheck<true>(this, nullcheck);
902             return newLenArray;
903         }
904     }
905     return nullptr;
906 }
907 
InsertDeoptimizationForIndexOverflow(CountableLoopInfo * countableLoopInfo,BoundsRange indexUpperRange,Inst * ss)908 void ChecksElimination::InsertDeoptimizationForIndexOverflow(CountableLoopInfo *countableLoopInfo,
909                                                              BoundsRange indexUpperRange, Inst *ss)
910 {
911     auto loopCc = countableLoopInfo->normalizedCc;
912     if (loopCc == CC_LT || loopCc == CC_LE) {
913         auto loopUpper = countableLoopInfo->test;
914         auto step = countableLoopInfo->constStep;
915         auto indexType = countableLoopInfo->index->GetType();
916         ASSERT(indexType == DataType::INT32);
917         auto maxUpper = BoundsRange::GetMax(indexType) - step + (loopCc == CC_LT ? 1 : 0);
918         auto bri = loopUpper->GetBasicBlock()->GetGraph()->GetBoundsRangeInfo();
919         auto loopUpperRange = bri->FindBoundsRange(countableLoopInfo->index->GetBasicBlock(), loopUpper);
920         // Upper bound of loop index assuming (index + max_add < len_array)
921         indexUpperRange = indexUpperRange.Add(BoundsRange(step)).FitInType(indexType);
922         if (!BoundsRange(maxUpper).IsMoreOrEqual(loopUpperRange) && indexUpperRange.IsMaxRange(indexType)) {
923             // loop index can overflow
924             Inst *insertAfter = loopUpper->IsDominate(ss) ? ss : loopUpper;
925             COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Build deoptimize for loop index overflow";
926             // Create deoptimize if loop index can become negative
927             InsertBoundsCheckDeoptimization(ConditionCode::CC_LT, nullptr, maxUpper, loopUpper, ss, insertAfter);
928         }
929     }
930 }
931 
NeedUpperDeoptimization(BasicBlock * header,Inst * lenArray,BoundsRange lenArrayRange,Inst * upper,BoundsRange upperRange,int64_t maxAdd,ConditionCode cc,bool * insertNewLenArray)932 bool ChecksElimination::NeedUpperDeoptimization(BasicBlock *header, Inst *lenArray, BoundsRange lenArrayRange,
933                                                 Inst *upper, BoundsRange upperRange, int64_t maxAdd, ConditionCode cc,
934                                                 bool *insertNewLenArray)
935 {
936     if (maxAdd == std::numeric_limits<int64_t>::min()) {
937         return false;
938     }
939     if (upperRange.Add(BoundsRange(maxAdd)).IsLess(lenArrayRange)) {
940         return false;
941     }
942     auto bri = GetGraph()->GetBoundsRangeInfo();
943     auto newUpper = upper;
944     auto newMaxAdd = maxAdd;
945     int64_t upperAdd = 0;
946     if (IsInstIncOrDec(upper)) {
947         upperAdd = GetInc(upper);
948         newMaxAdd += upperAdd;
949         newUpper = upper->GetInput(0).GetInst();
950     }
951     if (lenArray == newUpper) {
952         if (newMaxAdd < 0 || (newMaxAdd == 0 && cc == CC_LT)) {
953             return false;
954         }
955     }
956     auto useUpperLen = upperAdd >= 0 || !upperRange.IsMaxRange(upper->GetType());
957     if (useUpperLen && newMaxAdd <= 0) {
958         auto newUpperRange = bri->FindBoundsRange(header, newUpper);
959         if (newUpperRange.GetLenArray() == lenArray) {
960             return false;
961         }
962     }
963     *insertNewLenArray = newUpper != lenArray;
964     return true;
965 }
966 
TryInsertDeoptimizationForLargeStep(ConditionCode cc,Inst * resultLenArray,Inst * lower,Inst * upper,int64_t maxAdd,Inst * insertDeoptAfter,Inst * ss,uint64_t constStep)967 bool ChecksElimination::TryInsertDeoptimizationForLargeStep(ConditionCode cc, Inst *resultLenArray, Inst *lower,
968                                                             Inst *upper, int64_t maxAdd, Inst *insertDeoptAfter,
969                                                             Inst *ss, uint64_t constStep)
970 {
971     auto block = insertDeoptAfter->GetBasicBlock();
972     if (!lower->IsDominate(insertDeoptAfter)) {
973         if (lower->GetBasicBlock() == block) {
974             insertDeoptAfter = lower;
975         } else {
976             return false;
977         }
978     }
979     auto subValue = lower;
980     if (cc == CC_LT) {
981         subValue = GetGraph()->CreateInstAdd(DataType::INT32, INVALID_PC, lower, GetGraph()->FindOrCreateConstant(1));
982         InsertInstAfter(subValue, insertDeoptAfter, block);
983         insertDeoptAfter = subValue;
984     }
985     auto sub = GetGraph()->CreateInstSub(DataType::INT32, INVALID_PC, upper, subValue);
986     InsertInstAfter(sub, insertDeoptAfter, block);
987     auto mod = GetGraph()->CreateInstMod(DataType::INT32, INVALID_PC, sub, GetGraph()->FindOrCreateConstant(constStep));
988     block->InsertAfter(mod, sub);
989     if (resultLenArray == upper) {
990         auto maxAddConst = GetGraph()->FindOrCreateConstant(maxAdd);
991         // (upper - lower [- 1]) % step </<= max_add
992         InsertBoundsCheckDeoptimization(cc, mod, 0, maxAddConst, ss, mod, Opcode::NOP);
993     } else {
994         // result_len_array - max_add </<= upper - (upper - lower [- 1]) % step
995         auto maxIndexValue = GetGraph()->CreateInstSub(DataType::INT32, INVALID_PC, upper, mod);
996         block->InsertAfter(maxIndexValue, mod);
997         auto opcode = maxAdd > 0 ? Opcode::Sub : Opcode::SubOverflowCheck;
998         InsertBoundsCheckDeoptimization(cc, resultLenArray, maxAdd, maxIndexValue, ss, maxIndexValue, opcode);
999     }
1000     return true;
1001 }
1002 
TryInsertDeoptimization(LoopInfo loopInfo,Inst * lenArray,int64_t maxAdd,int64_t minAdd,bool hasCheckInHeader)1003 bool ChecksElimination::TryInsertDeoptimization(LoopInfo loopInfo, Inst *lenArray, int64_t maxAdd, int64_t minAdd,
1004                                                 bool hasCheckInHeader)
1005 {
1006     auto [countable_loop_info, ss, lower, upper, cc, is_head_loop_exit, has_pre_header_compare] = loopInfo;
1007     ASSERT(cc == CC_LT || cc == CC_LE);
1008     auto bri = GetGraph()->GetBoundsRangeInfo();
1009     auto header = countable_loop_info.index->GetBasicBlock();
1010     auto upperRange = bri->FindBoundsRange(header, upper);
1011     auto lowerRange = bri->FindBoundsRange(header, lower);
1012     auto lenArrayRange = bri->FindBoundsRange(header, lenArray);
1013     auto hasCheckBeforeExit = hasCheckInHeader || !is_head_loop_exit;
1014     if (!has_pre_header_compare && !lowerRange.IsLess(upperRange) && hasCheckBeforeExit) {
1015         // if lower > upper, removing BoundsCheck may be wrong for the first iteration
1016         return false;
1017     }
1018     uint64_t lowerInc = (countable_loop_info.normalizedCc == CC_GT ? 1 : 0);
1019     bool needLowerDeopt = (minAdd != std::numeric_limits<int64_t>::max()) &&
1020                           !lowerRange.Add(BoundsRange(minAdd)).Add(BoundsRange(lowerInc)).IsNotNegative();
1021     bool insertLowerDeopt = lower->IsDominate(ss);
1022     if (needLowerDeopt && !insertLowerDeopt) {
1023         COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Unable to build deoptimize for lower value";
1024         return false;
1025     }
1026 
1027     bool insertNewLenArray;
1028     if (NeedUpperDeoptimization(header, lenArray, lenArrayRange, upper, upperRange, maxAdd, cc, &insertNewLenArray)) {
1029         auto resultLenArray = insertNewLenArray ? InsertNewLenArray(lenArray, ss) : lenArray;
1030         if (resultLenArray == nullptr) {
1031             COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Unable to build deoptimize for upper value";
1032             return false;
1033         }
1034         auto insertDeoptAfter = lenArray != resultLenArray ? resultLenArray : ss;
1035         if (!upper->IsDominate(insertDeoptAfter)) {
1036             insertDeoptAfter = upper;
1037         }
1038         ASSERT(insertDeoptAfter->GetBasicBlock()->IsDominate(header));
1039         if (insertDeoptAfter->GetBasicBlock() == header) {
1040             COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Unable to build deoptimize for upper value";
1041             return false;
1042         }
1043         auto constStep = countable_loop_info.constStep;
1044         COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Try to build deoptimize for upper value";
1045         if (constStep == 1 ||
1046             (countable_loop_info.normalizedCc == CC_GT || countable_loop_info.normalizedCc == CC_GE)) {
1047             auto opcode = maxAdd > 0 ? Opcode::Sub : Opcode::SubOverflowCheck;
1048             // Create deoptimize if result_len_array - max_add <=(<) upper
1049             // result_len_array is >= 0, so if max_add > 0, overflow is not possible
1050             // that's why we do not add max_add to upper instead
1051             InsertBoundsCheckDeoptimization(cc, resultLenArray, maxAdd, upper, ss, insertDeoptAfter, opcode);
1052         } else if (lowerRange.IsConst() && lowerRange.GetLeft() == 0 && countable_loop_info.normalizedCc == CC_LT &&
1053                    resultLenArray == upper && maxAdd == static_cast<int64_t>(constStep) - 1) {
1054             // For (int i = 0; i < len; i += x) process(a[i], ..., a[i + x - 1])
1055             // deoptimize if len % x != 0
1056             auto zeroConst = GetGraph()->FindOrCreateConstant(0);
1057             InsertBoundsCheckDeoptimization(ConditionCode::CC_NE, resultLenArray, constStep, zeroConst, ss,
1058                                             insertDeoptAfter, Opcode::Mod);
1059         } else if (!TryInsertDeoptimizationForLargeStep(cc, resultLenArray, lower, upper, maxAdd, insertDeoptAfter, ss,
1060                                                         constStep)) {
1061             COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Unable to build deoptimize for upper value with step > 1";
1062             return false;
1063         }
1064     }
1065     InsertDeoptimizationForIndexOverflow(&countable_loop_info, lenArrayRange.Sub(BoundsRange(maxAdd)), ss);
1066     if (needLowerDeopt) {
1067         COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Build deoptimize for lower value";
1068         // Create deoptimize if lower < 0 (or -1 for loop with CC_GT)
1069         auto lowerConst = GetGraph()->FindOrCreateConstant(-lowerInc);
1070         InsertBoundsCheckDeoptimization(ConditionCode::CC_LT, lower, minAdd, lowerConst, ss, ss);
1071     }
1072     return true;
1073 }
1074 
HoistLoopInvariantBoundsChecks(Inst * lenArray,GroupedBoundsChecks * indexBoundschecks,Loop * loop)1075 void ChecksElimination::HoistLoopInvariantBoundsChecks(Inst *lenArray, GroupedBoundsChecks *indexBoundschecks,
1076                                                        Loop *loop)
1077 {
1078     auto lenArrLoop = lenArray->GetBasicBlock()->GetLoop();
1079     // len_array isn't loop invariant
1080     if (lenArrLoop == loop) {
1081         return;
1082     }
1083     for (auto &[index, bound_checks_info] : *indexBoundschecks) {
1084         // Check that index is loop invariant, if index is nullptr it means that it was a constant
1085         if (index != nullptr && index->GetBasicBlock()->GetLoop() == loop) {
1086             continue;
1087         }
1088         for (auto boundsCheck : std::get<0>(bound_checks_info)) {
1089             PushNewCheckForMoveOutOfLoop(boundsCheck);
1090         }
1091     }
1092 }
1093 
ProcessingGroupBoundsCheck(GroupedBoundsChecks * indexBoundschecks,LoopInfo loopInfo,Inst * lenArray)1094 void ChecksElimination::ProcessingGroupBoundsCheck(GroupedBoundsChecks *indexBoundschecks, LoopInfo loopInfo,
1095                                                    Inst *lenArray)
1096 {
1097     auto phiIndex = std::get<0>(loopInfo).index;
1098     if (indexBoundschecks->find(phiIndex) == indexBoundschecks->end()) {
1099         HoistLoopInvariantBoundsChecks(lenArray, indexBoundschecks, phiIndex->GetBasicBlock()->GetLoop());
1100         return;
1101     }
1102     auto &[insts_to_delete, max_add, min_add] = indexBoundschecks->at(phiIndex);
1103     ASSERT(!insts_to_delete.empty());
1104     bool hasCheckInHeader = false;
1105     for (const auto &inst : insts_to_delete) {
1106         if (inst->GetBasicBlock() == phiIndex->GetBasicBlock()) {
1107             hasCheckInHeader = true;
1108         }
1109     }
1110     if (TryInsertDeoptimization(loopInfo, lenArray, max_add, min_add, hasCheckInHeader)) {
1111         COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Delete group of BoundsChecks";
1112         // Delete bounds checks instructions
1113         for (const auto &inst : insts_to_delete) {
1114             ReplaceUsersAndRemoveCheck(inst, inst->GetInput(1).GetInst());
1115             if (indexBoundschecks->find(inst->GetInput(1).GetInst()) != indexBoundschecks->end()) {
1116                 indexBoundschecks->erase(inst->GetInput(1).GetInst());
1117             }
1118         }
1119     }
1120 }
1121 
ProcessingLoop(Loop * loop,ArenaUnorderedMap<Inst *,GroupedBoundsChecks> * lenarrIndexChecks)1122 void ChecksElimination::ProcessingLoop(Loop *loop, ArenaUnorderedMap<Inst *, GroupedBoundsChecks> *lenarrIndexChecks)
1123 {
1124     auto loopInfo = FindLoopInfo(loop);
1125     if (loopInfo == std::nullopt) {
1126         return;
1127     }
1128     for (auto &[len_array, index_boundschecks] : *lenarrIndexChecks) {
1129         ProcessingGroupBoundsCheck(&index_boundschecks, loopInfo.value(), len_array);
1130     }
1131 }
1132 
ReplaceBoundsCheckToDeoptimizationBeforeLoop()1133 void ChecksElimination::ReplaceBoundsCheckToDeoptimizationBeforeLoop()
1134 {
1135     COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Start ReplaceBoundsCheckToDeoptimizationBeforeLoop";
1136     for (auto &[loop, lenarr_index_checks] : boundsChecks_) {
1137         COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Processing loop with id = " << loop->GetId();
1138         if (loop->IsRoot()) {
1139             COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Skip root loop";
1140             continue;
1141         }
1142         ProcessingLoop(loop, &lenarr_index_checks);
1143     }
1144     COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Finish ReplaceBoundsCheckToDeoptimizationBeforeLoop";
1145 }
1146 
MoveCheckOutOfLoop()1147 void ChecksElimination::MoveCheckOutOfLoop()
1148 {
1149     for (auto inst : checksForMoveOutOfLoop_) {
1150         Inst *insertAfter = nullptr;
1151         auto ss = FindOptimalSaveStateForHoist(inst, &insertAfter);
1152         if (ss == nullptr) {
1153             continue;
1154         }
1155         ASSERT(insertAfter != nullptr);
1156         ASSERT(ss->GetBasicBlock() == insertAfter->GetBasicBlock());
1157         auto block = inst->GetBasicBlock();
1158         COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Move check " << GetOpcodeString(inst->GetOpcode())
1159                                          << " with id = " << inst->GetId() << " from bb " << block->GetId() << " to bb "
1160                                          << ss->GetBasicBlock()->GetId();
1161         block->EraseInst(inst);
1162         ss->GetBasicBlock()->InsertAfter(inst, insertAfter);
1163         inst->SetSaveState(ss);
1164         inst->SetPc(ss->GetPc());
1165         inst->SetFlag(inst_flags::CAN_DEOPTIMIZE);
1166         SetApplied();
1167     }
1168 }
1169 
FindSaveState(const InstVector & instsToDelete)1170 Inst *ChecksElimination::FindSaveState(const InstVector &instsToDelete)
1171 {
1172     for (auto boundsCheck : instsToDelete) {
1173         bool isDominate = true;
1174         for (auto boundsCheckTest : instsToDelete) {
1175             if (boundsCheck == boundsCheckTest) {
1176                 continue;
1177             }
1178             isDominate &= boundsCheck->IsDominate(boundsCheckTest);
1179         }
1180         if (isDominate) {
1181             constexpr auto IMM_2 = 2;
1182             return boundsCheck->GetInput(IMM_2).GetInst();
1183         }
1184     }
1185     return nullptr;
1186 }
1187 
ReplaceBoundsCheckToDeoptimizationInLoop()1188 void ChecksElimination::ReplaceBoundsCheckToDeoptimizationInLoop()
1189 {
1190     COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Start ReplaceBoundsCheckToDeoptimizationInLoop";
1191     for (auto &item : boundsChecks_) {
1192         for (auto &[len_array, index_boundschecks_map] : item.second) {
1193             for (auto &[index, it] : index_boundschecks_map) {
1194                 auto [insts_to_delete, max, min] = it;
1195                 constexpr auto MIN_BOUNDSCHECKS_NUM = 2;
1196                 if (insts_to_delete.size() <= MIN_BOUNDSCHECKS_NUM) {
1197                     COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Skip small group of BoundsChecks";
1198                     continue;
1199                 }
1200                 // Try to replace more than 2 bounds checks to deoptimization in loop
1201                 auto saveState = FindSaveState(insts_to_delete);
1202                 if (saveState == nullptr) {
1203                     COMPILER_LOG(DEBUG, CHECKS_ELIM) << "SaveState isn't founded";
1204                     continue;
1205                 }
1206                 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Replace group of BoundsChecks on deoptimization in loop";
1207                 auto insertAfter = len_array->IsDominate(saveState) ? saveState : len_array;
1208                 if (max != std::numeric_limits<int64_t>::min()) {
1209                     // Create deoptimize if max_index >= len_array
1210                     InsertBoundsCheckDeoptimization(ConditionCode::CC_GE, index, max, len_array, saveState,
1211                                                     insertAfter);
1212                 }
1213                 if (index != nullptr && min != std::numeric_limits<int64_t>::max()) {
1214                     // Create deoptimize if min_index < 0
1215                     auto zeroConst = GetGraph()->FindOrCreateConstant(0);
1216                     InsertBoundsCheckDeoptimization(ConditionCode::CC_LT, index, min, zeroConst, saveState,
1217                                                     insertAfter);
1218                 } else {
1219                     // No lower check needed based on BoundsAnalysis
1220                     // if index is null, group of bounds checks consists of constants
1221                     ASSERT(min >= 0);
1222                 }
1223                 for (const auto &inst : insts_to_delete) {
1224                     COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Delete group of BoundsChecks";
1225                     ReplaceUsersAndRemoveCheck(inst, inst->GetInput(1).GetInst());
1226                 }
1227             }
1228         }
1229     }
1230     COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Finish ReplaceBoundsCheckToDeoptimizationInLoop";
1231 }
1232 
ReplaceCheckMustThrowByUnconditionalDeoptimize()1233 void ChecksElimination::ReplaceCheckMustThrowByUnconditionalDeoptimize()
1234 {
1235     for (auto &inst : checksMustThrow_) {
1236         auto block = inst->GetBasicBlock();
1237         if (block != nullptr) {
1238             COMPILER_LOG(DEBUG, CHECKS_ELIM)
1239                 << "Replace check with id = " << inst->GetId() << " by uncondition deoptimize";
1240             block->ReplaceInstByDeoptimize(inst);
1241             SetApplied();
1242             SetLoopDeleted();
1243         }
1244     }
1245 }
1246 
1247 }  // namespace panda::compiler
1248