1 /*
2 * Copyright (c) 2021-2023 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "compiler_logger.h"
17 #include "checks_elimination.h"
18 #include "optimizer/analysis/alias_analysis.h"
19 #include "optimizer/analysis/bounds_analysis.h"
20 #include "optimizer/analysis/dominators_tree.h"
21 #include "optimizer/analysis/loop_analyzer.h"
22 #include "optimizer/analysis/object_type_propagation.h"
23 #include "optimizer/ir/graph_visitor.h"
24 #include "optimizer/ir/analysis.h"
25
26 namespace panda::compiler {
27
RunImpl()28 bool ChecksElimination::RunImpl()
29 {
30 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Start ChecksElimination";
31 GetGraph()->RunPass<DominatorsTree>();
32 GetGraph()->RunPass<LoopAnalyzer>();
33 GetGraph()->RunPass<ObjectTypePropagation>();
34
35 VisitGraph();
36
37 if (g_options.IsCompilerEnableReplacingChecksOnDeoptimization()) {
38 if (!GetGraph()->IsOsrMode()) {
39 ReplaceBoundsCheckToDeoptimizationBeforeLoop();
40 MoveCheckOutOfLoop();
41 }
42 ReplaceBoundsCheckToDeoptimizationInLoop();
43
44 ReplaceCheckMustThrowByUnconditionalDeoptimize();
45 }
46 if (IsLoopDeleted() && GetGraph()->IsOsrMode()) {
47 CleanupGraphSaveStateOSR(GetGraph());
48 }
49 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "ChecksElimination " << (IsApplied() ? "is" : "isn't") << " applied";
50 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Finish ChecksElimination";
51 return isApplied_;
52 }
53
InvalidateAnalyses()54 void ChecksElimination::InvalidateAnalyses()
55 {
56 GetGraph()->InvalidateAnalysis<DominatorsTree>();
57 // Already "LoopAnalyzer" was ran in "CleanupGraphSaveStateOSR"
58 // in case (IsLoopDeleted() && GetGraph()->IsOsrMode())
59 if (!(IsLoopDeleted() && GetGraph()->IsOsrMode())) {
60 GetGraph()->InvalidateAnalysis<LoopAnalyzer>();
61 }
62 GetGraph()->InvalidateAnalysis<BoundsAnalysis>();
63 GetGraph()->InvalidateAnalysis<AliasAnalysis>();
64 }
65
66 template <bool CHECK_FULL_DOM>
VisitNullCheck(GraphVisitor * v,Inst * inst)67 void ChecksElimination::VisitNullCheck(GraphVisitor *v, Inst *inst)
68 {
69 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Start visit NullCheck with id = " << inst->GetId();
70
71 auto ref = inst->GetInput(0).GetInst();
72 static_cast<ChecksElimination *>(v)->TryRemoveDominatedNullChecks(inst, ref);
73
74 if (!static_cast<ChecksElimination *>(v)->TryRemoveCheck<Opcode::NullCheck, CHECK_FULL_DOM>(inst)) {
75 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "NullCheck couldn't be deleted";
76 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "NullCheck saved for further replacing on deoptimization";
77 static_cast<ChecksElimination *>(v)->PushNewCheckForMoveOutOfLoop(inst);
78 }
79 }
80
VisitNegativeCheck(GraphVisitor * v,Inst * inst)81 void ChecksElimination::VisitNegativeCheck(GraphVisitor *v, Inst *inst)
82 {
83 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Start visit NegativeCheck with id = " << inst->GetId();
84 if (!static_cast<ChecksElimination *>(v)->TryRemoveCheck<Opcode::NegativeCheck>(inst)) {
85 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "NegativeCheck couldn't be deleted";
86 static_cast<ChecksElimination *>(v)->PushNewCheckForMoveOutOfLoop(inst);
87 }
88 }
89
VisitNotPositiveCheck(GraphVisitor * v,Inst * inst)90 void ChecksElimination::VisitNotPositiveCheck(GraphVisitor *v, Inst *inst)
91 {
92 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Start visit NotPositiveCheck with id = " << inst->GetId();
93 if (!static_cast<ChecksElimination *>(v)->TryRemoveCheck<Opcode::NotPositiveCheck>(inst)) {
94 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "NotPositiveCheck couldn't be deleted";
95 static_cast<ChecksElimination *>(v)->PushNewCheckForMoveOutOfLoop(inst);
96 }
97 }
98
VisitZeroCheck(GraphVisitor * v,Inst * inst)99 void ChecksElimination::VisitZeroCheck(GraphVisitor *v, Inst *inst)
100 {
101 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Start visit ZeroCheck with id = " << inst->GetId();
102 if (!static_cast<ChecksElimination *>(v)->TryRemoveCheck<Opcode::ZeroCheck>(inst)) {
103 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "ZeroCheck couldn't be deleted";
104 static_cast<ChecksElimination *>(v)->PushNewCheckForMoveOutOfLoop(inst);
105 }
106 }
107
VisitRefTypeCheck(GraphVisitor * v,Inst * inst)108 void ChecksElimination::VisitRefTypeCheck(GraphVisitor *v, Inst *inst)
109 {
110 auto visitor = static_cast<ChecksElimination *>(v);
111 auto storeInst = inst->GetDataFlowInput(inst->GetInput(1).GetInst());
112 // Case: a[i] = nullptr
113 if (storeInst->GetOpcode() == Opcode::NullPtr) {
114 visitor->ReplaceUsersAndRemoveCheck(inst, storeInst);
115 return;
116 }
117 auto arrayInst = inst->GetDataFlowInput(0);
118 auto ref = inst->GetInput(1).GetInst();
119 // Case:
120 // a[1] = obj
121 // a[2] = obj
122 visitor->TryRemoveDominatedChecks<Opcode::RefTypeCheck>(inst, [arrayInst, ref](Inst *userInst) {
123 return userInst->GetDataFlowInput(0) == arrayInst && userInst->GetInput(1) == ref;
124 });
125 visitor->GetGraph()->RunPass<ObjectTypePropagation>();
126 auto arrayTypeInfo = arrayInst->GetObjectTypeInfo();
127 if (arrayTypeInfo && arrayTypeInfo.IsExact()) {
128 auto storeTypeInfo = storeInst->GetObjectTypeInfo();
129 auto arrayClass = arrayTypeInfo.GetClass();
130 auto storeClass = (storeTypeInfo) ? storeTypeInfo.GetClass() : nullptr;
131 if (visitor->GetGraph()->GetRuntime()->CheckStoreArray(arrayClass, storeClass)) {
132 visitor->ReplaceUsersAndRemoveCheck(inst, storeInst);
133 }
134 }
135 }
136
TryToEliminateAnyTypeCheck(Inst * inst,Inst * instToReplace,AnyBaseType type,AnyBaseType prevType)137 bool ChecksElimination::TryToEliminateAnyTypeCheck(Inst *inst, Inst *instToReplace, AnyBaseType type,
138 AnyBaseType prevType)
139 {
140 auto language = GetGraph()->GetRuntime()->GetMethodSourceLanguage(GetGraph()->GetMethod());
141 auto allowedType = inst->CastToAnyTypeCheck()->GetAllowedInputType();
142 profiling::AnyInputType prevAllowedType;
143 if (instToReplace->GetOpcode() == Opcode::AnyTypeCheck) {
144 prevAllowedType = instToReplace->CastToAnyTypeCheck()->GetAllowedInputType();
145 } else {
146 prevAllowedType = instToReplace->CastToCastValueToAnyType()->GetAllowedInputType();
147 }
148 auto res = IsAnyTypeCanBeSubtypeOf(language, type, prevType, allowedType, prevAllowedType);
149 if (!res) {
150 return false;
151 }
152 if (res.value()) {
153 ReplaceUsersAndRemoveCheck(inst, instToReplace);
154 } else {
155 PushNewCheckMustThrow(inst);
156 }
157 return true;
158 }
159
UpdateHclassChecks(Inst * inst)160 void ChecksElimination::UpdateHclassChecks(Inst *inst)
161 {
162 // AnyTypeCheck HEAP_OBJECT => LoadObject Class => LoadObject Hclass => HclassCheck
163 bool allUsersInlined = false;
164 for (auto &user : inst->GetUsers()) {
165 auto userInst = user.GetInst();
166 if (userInst->IsCall()) {
167 if (userInst->CastToCallDynamic()->IsInlined()) {
168 allUsersInlined = true;
169 } else {
170 return;
171 }
172 } else if (userInst->GetOpcode() == Opcode::LoadObject) {
173 if (userInst->CastToLoadObject()->GetObjectType() == ObjectType::MEM_DYN_CLASS) {
174 continue;
175 }
176 ASSERT(userInst->CastToLoadObject()->GetObjectType() == ObjectType::MEM_DYN_METHOD);
177 if (!IsInlinedCallLoadMethod(userInst)) {
178 return;
179 }
180 allUsersInlined = true;
181 } else {
182 return;
183 }
184 }
185 if (!allUsersInlined) {
186 return;
187 }
188 for (auto &user : inst->GetUsers()) {
189 auto userInst = user.GetInst();
190 auto check = GetHclassCheckFromLoads(userInst);
191 if (!check.has_value()) {
192 continue;
193 }
194 check.value()->CastToHclassCheck()->SetCheckFunctionIsNotClassConstructor(false);
195 SetApplied();
196 }
197 }
198
GetHclassCheckFromLoads(Inst * loadClass)199 std::optional<Inst *> ChecksElimination::GetHclassCheckFromLoads(Inst *loadClass)
200 {
201 if (loadClass->GetOpcode() != Opcode::LoadObject ||
202 loadClass->CastToLoadObject()->GetObjectType() != ObjectType::MEM_DYN_CLASS) {
203 return std::nullopt;
204 }
205 for (auto &usersLoad : loadClass->GetUsers()) {
206 auto loadHclass = usersLoad.GetInst();
207 if (loadHclass->GetOpcode() != Opcode::LoadObject ||
208 loadHclass->CastToLoadObject()->GetObjectType() != ObjectType::MEM_DYN_HCLASS) {
209 continue;
210 }
211 for (auto &usersHclass : loadHclass->GetUsers()) {
212 auto hclassCheck = usersHclass.GetInst();
213 if (hclassCheck->GetOpcode() == Opcode::HclassCheck) {
214 return hclassCheck;
215 }
216 }
217 }
218 return std::nullopt;
219 }
220
IsInlinedCallLoadMethod(Inst * inst)221 bool ChecksElimination::IsInlinedCallLoadMethod(Inst *inst)
222 {
223 bool isMethod = inst->GetOpcode() == Opcode::LoadObject &&
224 inst->CastToLoadObject()->GetObjectType() == ObjectType::MEM_DYN_METHOD;
225 if (!isMethod) {
226 return false;
227 }
228 for (auto &methodUser : inst->GetUsers()) {
229 auto compare = methodUser.GetInst()->CastToCompare();
230 bool isCompare = compare->GetOpcode() == Opcode::Compare && compare->GetCc() == ConditionCode::CC_NE &&
231 compare->GetInputType(0) == DataType::POINTER;
232 if (!isCompare) {
233 return false;
234 }
235 auto deoptimizeIf = compare->GetFirstUser()->GetInst()->CastToDeoptimizeIf();
236 if (deoptimizeIf->GetDeoptimizeType() != DeoptimizeType::INLINE_DYN) {
237 return false;
238 }
239 }
240 return true;
241 }
242
TryRemoveDominatedHclassCheck(Inst * inst)243 void ChecksElimination::TryRemoveDominatedHclassCheck(Inst *inst)
244 {
245 ASSERT(inst->GetOpcode() == Opcode::HclassCheck);
246 // AnyTypeCheck HEAP_OBJECT => LoadObject Class => LoadObject Hclass => HclassCheck
247 auto object = inst->GetInput(0).GetInst()->GetInput(0).GetInst()->GetInput(0).GetInst();
248
249 for (auto &user : object->GetUsers()) {
250 auto userInst = user.GetInst();
251 auto hclassCheck = GetHclassCheckFromLoads(userInst);
252 if (hclassCheck.has_value() && hclassCheck.value() != inst && inst->IsDominate(hclassCheck.value())) {
253 ASSERT(inst->IsDominate(hclassCheck.value()));
254 COMPILER_LOG(DEBUG, CHECKS_ELIM)
255 << GetOpcodeString(Opcode::HclassCheck) << " with id = " << inst->GetId() << " dominate on "
256 << GetOpcodeString(Opcode::HclassCheck) << " with id = " << hclassCheck.value()->GetId();
257 auto block = hclassCheck.value()->GetBasicBlock();
258 auto graph = block->GetGraph();
259 hclassCheck.value()->RemoveInputs();
260 // We don't should update because it's possible to delete one flag in check, when all DynCall is inlined
261 inst->CastToHclassCheck()->ExtendFlags(hclassCheck.value());
262 block->ReplaceInst(hclassCheck.value(), graph->CreateInstNOP());
263 SetApplied();
264 }
265 }
266 }
267
VisitAnyTypeCheck(GraphVisitor * v,Inst * inst)268 void ChecksElimination::VisitAnyTypeCheck(GraphVisitor *v, Inst *inst)
269 {
270 auto visitor = static_cast<ChecksElimination *>(v);
271 auto inputInst = inst->GetInput(0).GetInst();
272 auto type = inst->CastToAnyTypeCheck()->GetAnyType();
273 if (type == AnyBaseType::UNDEFINED_TYPE) {
274 visitor->ReplaceUsersAndRemoveCheck(inst, inputInst);
275 return;
276 }
277 // from:
278 // 2.any CastValueToAnyType ANY_SUBTYPE v1 -> (v4)
279 // 4.any AnyTypeCheck ANY_SUBTYPE v2, v3 -> (....)
280 // to:
281 // 2.any CastValueToAnyType ANY_SUBTYPE v1 -> (...)
282 if (inputInst->GetOpcode() == Opcode::CastValueToAnyType) {
283 visitor->TryToEliminateAnyTypeCheck(inst, inputInst, type, inputInst->CastToCastValueToAnyType()->GetAnyType());
284 return;
285 }
286 // from:
287 // 2.any AnyTypeCheck ANY_SUBTYPE v1, v0 -> (v4)
288 // 4.any AnyTypeCheck ANY_SUBTYPE v2, v3 -> (....)
289 // to:
290 // 2.any AnyTypeCheck ANY_SUBTYPE v1, v0 -> (...)
291 if (inputInst->GetOpcode() == Opcode::AnyTypeCheck) {
292 visitor->TryToEliminateAnyTypeCheck(inst, inputInst, type, inputInst->CastToAnyTypeCheck()->GetAnyType());
293 return;
294 }
295 // from:
296 // 2.any AnyTypeCheck ANY_SUBTYPE v1, v0 -> (v4)
297 // 4.any AnyTypeCheck ANY_SUBTYPE v1, v3 -> (....)
298 // to:
299 // 2.any AnyTypeCheck ANY_SUBTYPE v1, v0 -> (v4,...)
300 bool applied = false;
301 for (auto &user : inputInst->GetUsers()) {
302 auto userInst = user.GetInst();
303 if (userInst == inst) {
304 continue;
305 }
306 if (userInst->GetOpcode() != Opcode::AnyTypeCheck) {
307 continue;
308 }
309 if (!inst->IsDominate(userInst)) {
310 continue;
311 }
312
313 if (visitor->TryToEliminateAnyTypeCheck(userInst, inst, userInst->CastToAnyTypeCheck()->GetAnyType(), type)) {
314 applied = true;
315 }
316 }
317 if (!applied) {
318 visitor->PushNewCheckForMoveOutOfLoop(inst);
319 }
320 // from:
321 // 39.any AnyTypeCheck ECMASCRIPT_HEAP_OBJECT_TYPE v37, v38 -> (v65, v40)
322 // 40.ref LoadObject 4294967288 Class v39 -> (v41)
323 // 41.ref LoadObject 4294967287 Hclass v40
324 // 42. HclassCheck [IsFunc, IsNotClassConstr] v41, v38
325 // ...METHOD IS INLINED ...
326 // to:
327 // 39.any AnyTypeCheck ECMASCRIPT_HEAP_OBJECT_TYPE v37, v38 -> (v65, v40)
328 // 40.ref LoadObject 4294967288 Class v39 -> (v41)
329 // 41.ref LoadObject 4294967287 Hclass v40
330 // 42. HclassCheck [IsFunc] v41, v38
331 // ...METHOD IS INLINED ...
332 if (TryIsDynHeapObject(type)) {
333 visitor->UpdateHclassChecks(inst);
334 }
335 }
336
VisitHclassCheck(GraphVisitor * v,Inst * inst)337 void ChecksElimination::VisitHclassCheck(GraphVisitor *v, Inst *inst)
338 {
339 [[maybe_unused]] auto visitor = static_cast<ChecksElimination *>(v);
340 visitor->TryRemoveDominatedHclassCheck(inst);
341 visitor->PushNewCheckForMoveOutOfLoop(inst);
342 }
343
VisitBoundsCheck(GraphVisitor * v,Inst * inst)344 void ChecksElimination::VisitBoundsCheck(GraphVisitor *v, Inst *inst)
345 {
346 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Start visit BoundsCheck with id = " << inst->GetId();
347 auto block = inst->GetBasicBlock();
348 auto lenArray = inst->GetInput(0).GetInst();
349 auto index = inst->GetInput(1).GetInst();
350 auto visitor = static_cast<ChecksElimination *>(v);
351
352 visitor->TryRemoveDominatedChecks<Opcode::BoundsCheck>(inst, [lenArray, index](Inst *userInst) {
353 return lenArray == userInst->GetInput(0) && index == userInst->GetInput(1);
354 });
355
356 if (index->GetType() == DataType::UINT64) {
357 return;
358 }
359 auto bri = block->GetGraph()->GetBoundsRangeInfo();
360 auto lenArrayRange = bri->FindBoundsRange(block, lenArray);
361 auto indexRange = bri->FindBoundsRange(block, index);
362 auto correctUpper = indexRange.IsLess(lenArrayRange) || indexRange.IsLess(lenArray);
363 auto correctLower = indexRange.IsNotNegative();
364 if (correctUpper && correctLower) {
365 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Index of BoundsCheck have correct bounds";
366 visitor->ReplaceUsersAndRemoveCheck(inst, index);
367 return;
368 }
369 if (indexRange.IsNegative() || indexRange.IsMoreOrEqual(lenArrayRange)) {
370 COMPILER_LOG(DEBUG, CHECKS_ELIM)
371 << "BoundsCheck have incorrect bounds, saved for replace by unconditional deoptimize";
372 visitor->PushNewCheckMustThrow(inst);
373 return;
374 }
375
376 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "BoundsCheck saved for further replacing on deoptimization";
377 auto loop = GetLoopForBoundsCheck(block, lenArray, index);
378 visitor->PushNewBoundsCheck(loop, lenArray, index, inst, !correctUpper, !correctLower);
379 }
380
VisitCheckCast(GraphVisitor * v,Inst * inst)381 void ChecksElimination::VisitCheckCast(GraphVisitor *v, Inst *inst)
382 {
383 auto visitor = static_cast<ChecksElimination *>(v);
384 visitor->GetGraph()->RunPass<ObjectTypePropagation>();
385 auto result = ObjectTypeCheckElimination::TryEliminateCheckCast(inst);
386 if (result != ObjectTypeCheckElimination::CheckCastEliminateType::INVALID) {
387 visitor->SetApplied();
388 if (result == ObjectTypeCheckElimination::CheckCastEliminateType::MUST_THROW) {
389 visitor->PushNewCheckMustThrow(inst);
390 }
391 return;
392 }
393 if (inst->CastToCheckCast()->GetOmitNullCheck()) {
394 return;
395 }
396 auto block = inst->GetBasicBlock();
397 auto bri = block->GetGraph()->GetBoundsRangeInfo();
398 auto inputRange = bri->FindBoundsRange(block, inst->GetInput(0).GetInst());
399 if (inputRange.IsMore(BoundsRange(0))) {
400 visitor->SetApplied();
401 inst->CastToCheckCast()->SetOmitNullCheck(true);
402 }
403 }
404
VisitIsInstance(GraphVisitor * v,Inst * inst)405 void ChecksElimination::VisitIsInstance(GraphVisitor *v, Inst *inst)
406 {
407 if (inst->CastToIsInstance()->GetOmitNullCheck()) {
408 return;
409 }
410 auto block = inst->GetBasicBlock();
411 auto bri = block->GetGraph()->GetBoundsRangeInfo();
412 auto inputRange = bri->FindBoundsRange(block, inst->GetInput(0).GetInst());
413 if (inputRange.IsMore(BoundsRange(0))) {
414 auto visitor = static_cast<ChecksElimination *>(v);
415 visitor->SetApplied();
416 inst->CastToIsInstance()->SetOmitNullCheck(true);
417 }
418 }
419
VisitAddOverflowCheck(GraphVisitor * v,Inst * inst)420 void ChecksElimination::VisitAddOverflowCheck(GraphVisitor *v, Inst *inst)
421 {
422 ASSERT(inst->GetType() == DataType::INT32);
423 auto input1 = inst->GetInput(0).GetInst();
424 auto input2 = inst->GetInput(1).GetInst();
425 auto visitor = static_cast<ChecksElimination *>(v);
426 visitor->TryRemoveDominatedChecks<Opcode::AddOverflowCheck>(inst, [input1, input2](Inst *userInst) {
427 return (userInst->GetInput(0) == input1 && userInst->GetInput(1) == input2) ||
428 (userInst->GetInput(0) == input2 && userInst->GetInput(1) == input1);
429 });
430 visitor->TryOptimizeOverflowCheck<Opcode::AddOverflowCheck>(inst);
431 }
432
VisitSubOverflowCheck(GraphVisitor * v,Inst * inst)433 void ChecksElimination::VisitSubOverflowCheck(GraphVisitor *v, Inst *inst)
434 {
435 ASSERT(inst->GetType() == DataType::INT32);
436 auto input1 = inst->GetInput(0).GetInst();
437 auto input2 = inst->GetInput(1).GetInst();
438 auto visitor = static_cast<ChecksElimination *>(v);
439 visitor->TryRemoveDominatedChecks<Opcode::SubOverflowCheck>(inst, [input1, input2](Inst *userInst) {
440 return (userInst->GetInput(0) == input1 && userInst->GetInput(1) == input2);
441 });
442 visitor->TryOptimizeOverflowCheck<Opcode::SubOverflowCheck>(inst);
443 }
444
VisitNegOverflowAndZeroCheck(GraphVisitor * v,Inst * inst)445 void ChecksElimination::VisitNegOverflowAndZeroCheck(GraphVisitor *v, Inst *inst)
446 {
447 ASSERT(inst->GetType() == DataType::INT32);
448 auto input1 = inst->GetInput(0).GetInst();
449 auto visitor = static_cast<ChecksElimination *>(v);
450 visitor->TryRemoveDominatedChecks<Opcode::NegOverflowAndZeroCheck>(
451 inst, [input1](Inst *userInst) { return (userInst->GetInput(0) == input1); });
452 visitor->TryOptimizeOverflowCheck<Opcode::NegOverflowAndZeroCheck>(inst);
453 }
454
ReplaceUsersAndRemoveCheck(Inst * instDel,Inst * instRep)455 void ChecksElimination::ReplaceUsersAndRemoveCheck(Inst *instDel, Inst *instRep)
456 {
457 auto block = instDel->GetBasicBlock();
458 auto graph = block->GetGraph();
459 if (graph->IsOsrMode() && block->GetLoop() != instRep->GetBasicBlock()->GetLoop()) {
460 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Check couldn't be deleted, becouse in OSR mode we can't replace "
461 "instructions with instructions from another loop";
462 return;
463 }
464 instDel->ReplaceUsers(instRep);
465 instDel->RemoveInputs();
466 block->ReplaceInst(instDel, graph->CreateInstNOP());
467 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Checks elimination delete " << GetOpcodeString(instDel->GetOpcode())
468 << " with id " << instDel->GetId();
469 graph->GetEventWriter().EventChecksElimination(GetOpcodeString(instDel->GetOpcode()), instDel->GetId(),
470 instDel->GetPc());
471 SetApplied();
472 }
473
IsInstIncOrDec(Inst * inst)474 bool ChecksElimination::IsInstIncOrDec(Inst *inst)
475 {
476 return inst->IsAddSub() && inst->GetInput(1).GetInst()->IsConst();
477 }
478
GetInc(Inst * inst)479 int64_t ChecksElimination::GetInc(Inst *inst)
480 {
481 ASSERT(IsInstIncOrDec(inst));
482 auto val = static_cast<int64_t>(inst->GetInput(1).GetInst()->CastToConstant()->GetIntValue());
483 if (inst->IsSub()) {
484 val = -val;
485 }
486 return val;
487 }
488
GetLoopForBoundsCheck(BasicBlock * block,Inst * lenArray,Inst * index)489 Loop *ChecksElimination::GetLoopForBoundsCheck(BasicBlock *block, Inst *lenArray, Inst *index)
490 {
491 auto parentIndex = IsInstIncOrDec(index) ? index->GetInput(0).GetInst() : index;
492 auto indexBlock = parentIndex->GetBasicBlock();
493 ASSERT(indexBlock != nullptr);
494 auto indexLoop = indexBlock->GetLoop();
495 if (auto loopInfo = CountableLoopParser(*indexLoop).Parse()) {
496 auto input = lenArray;
497 if (lenArray->GetOpcode() == Opcode::LenArray) {
498 // new len_array can be inserted
499 input = lenArray->GetDataFlowInput(0);
500 }
501 if (loopInfo->index == parentIndex && input->GetBasicBlock()->IsDominate(indexBlock)) {
502 ASSERT(indexBlock == indexLoop->GetHeader());
503 return indexLoop;
504 }
505 }
506 return block->GetLoop();
507 }
508
InitItemForNewIndex(GroupedBoundsChecks * place,Inst * index,Inst * inst,bool checkUpper,bool checkLower)509 void ChecksElimination::InitItemForNewIndex(GroupedBoundsChecks *place, Inst *index, Inst *inst, bool checkUpper,
510 bool checkLower)
511 {
512 ASSERT(inst->GetOpcode() == Opcode::BoundsCheck);
513 InstVector insts(GetGraph()->GetLocalAllocator()->Adapter());
514 insts.push_back(inst);
515 int64_t val = 0;
516 Inst *parentIndex = index;
517 if (IsInstIncOrDec(index)) {
518 val = GetInc(index);
519 parentIndex = index->GetInput(0).GetInst();
520 } else if (index->IsConst()) {
521 val = static_cast<int64_t>(index->CastToConstant()->GetIntValue());
522 parentIndex = nullptr;
523 }
524 auto maxVal = checkUpper ? val : std::numeric_limits<int64_t>::min();
525 auto minVal = checkLower ? val : std::numeric_limits<int64_t>::max();
526 place->emplace(parentIndex, std::make_tuple(insts, maxVal, minVal));
527 }
528
PushNewBoundsCheck(Loop * loop,Inst * lenArray,Inst * index,Inst * inst,bool checkUpper,bool checkLower)529 void ChecksElimination::PushNewBoundsCheck(Loop *loop, Inst *lenArray, Inst *index, Inst *inst, bool checkUpper,
530 bool checkLower)
531 {
532 ASSERT(loop != nullptr && lenArray != nullptr && index != nullptr && inst != nullptr);
533 ASSERT(inst->GetOpcode() == Opcode::BoundsCheck);
534 if (boundsChecks_.find(loop) == boundsChecks_.end()) {
535 auto it1 = boundsChecks_.emplace(loop, GetGraph()->GetLocalAllocator()->Adapter());
536 ASSERT(it1.second);
537 auto it2 = it1.first->second.emplace(lenArray, GetGraph()->GetLocalAllocator()->Adapter());
538 ASSERT(it2.second);
539 InitItemForNewIndex(&it2.first->second, index, inst, checkUpper, checkLower);
540 } else if (boundsChecks_.at(loop).find(lenArray) == boundsChecks_.at(loop).end()) {
541 auto it1 = boundsChecks_.at(loop).emplace(lenArray, GetGraph()->GetLocalAllocator()->Adapter());
542 ASSERT(it1.second);
543 InitItemForNewIndex(&it1.first->second, index, inst, checkUpper, checkLower);
544 } else if (auto &lenABc = boundsChecks_.at(loop).at(lenArray); lenABc.find(index) == lenABc.end()) {
545 auto parentIndex = index;
546 int64_t val {};
547 if (IsInstIncOrDec(index)) {
548 parentIndex = index->GetInput(0).GetInst();
549 val = GetInc(index);
550 } else if (index->IsConst()) {
551 parentIndex = nullptr;
552 val = static_cast<int64_t>(index->CastToConstant()->GetIntValue());
553 }
554 if (parentIndex == index || lenABc.find(parentIndex) == lenABc.end()) {
555 InitItemForNewIndex(&lenABc, index, inst, checkUpper, checkLower);
556 } else {
557 auto &item = lenABc.at(parentIndex);
558 std::get<0>(item).push_back(inst);
559 if (val > std::get<1>(item) && checkUpper) {
560 std::get<1>(item) = val;
561 } else if (val < std::get<2U>(item) && checkLower) {
562 std::get<2U>(item) = val;
563 }
564 }
565 } else {
566 auto &item = boundsChecks_.at(loop).at(lenArray).at(index);
567 std::get<0>(item).push_back(inst);
568 if (std::get<1>(item) < 0 && checkUpper) {
569 std::get<1>(item) = 0;
570 }
571 if (std::get<2U>(item) > 0 && checkLower) {
572 std::get<2U>(item) = 0;
573 }
574 }
575 }
576
TryRemoveDominatedNullChecks(Inst * inst,Inst * ref)577 void ChecksElimination::TryRemoveDominatedNullChecks(Inst *inst, Inst *ref)
578 {
579 for (auto &user : ref->GetUsers()) {
580 auto userInst = user.GetInst();
581 if (((userInst->GetOpcode() == Opcode::IsInstance && !userInst->CastToIsInstance()->GetOmitNullCheck()) ||
582 (userInst->GetOpcode() == Opcode::CheckCast && !userInst->CastToCheckCast()->GetOmitNullCheck())) &&
583 inst->IsDominate(userInst)) {
584 COMPILER_LOG(DEBUG, CHECKS_ELIM)
585 << "NullCheck with id = " << inst->GetId() << " dominate on " << GetOpcodeString(userInst->GetOpcode())
586 << " with id = " << userInst->GetId();
587 if (userInst->GetOpcode() == Opcode::IsInstance) {
588 userInst->CastToIsInstance()->SetOmitNullCheck(true);
589 } else {
590 userInst->CastToCheckCast()->SetOmitNullCheck(true);
591 }
592 SetApplied();
593 }
594 }
595 }
596
597 template <Opcode OPC, bool CHECK_FULL_DOM, typename CheckInputs>
TryRemoveDominatedChecks(Inst * inst,CheckInputs checkInputs)598 void ChecksElimination::TryRemoveDominatedChecks(Inst *inst, CheckInputs checkInputs)
599 {
600 for (auto &user : inst->GetInput(0).GetInst()->GetUsers()) {
601 auto userInst = user.GetInst();
602 // NOLINTNEXTLINE(readability-magic-numbers)
603 if (userInst->GetOpcode() == OPC && userInst != inst && userInst->GetType() == inst->GetType() &&
604 checkInputs(userInst) &&
605 (CHECK_FULL_DOM ? inst->IsDominate(userInst) : inst->InSameBlockOrDominate(userInst))) {
606 ASSERT(inst->IsDominate(userInst));
607 COMPILER_LOG(DEBUG, CHECKS_ELIM)
608 // NOLINTNEXTLINE(readability-magic-numbers)
609 << GetOpcodeString(OPC) << " with id = " << inst->GetId() << " dominate on " << GetOpcodeString(OPC)
610 << " with id = " << userInst->GetId();
611 ReplaceUsersAndRemoveCheck(userInst, inst);
612 }
613 }
614 }
615
616 // Remove consecutive checks: NullCheck -> NullCheck -> NullCheck
617 template <Opcode OPC>
TryRemoveConsecutiveChecks(Inst * inst)618 void ChecksElimination::TryRemoveConsecutiveChecks(Inst *inst)
619 {
620 auto end = inst->GetUsers().end();
621 for (auto user = inst->GetUsers().begin(); user != end;) {
622 auto userInst = (*user).GetInst();
623 // NOLINTNEXTLINE(readability-magic-numbers)
624 if (userInst->GetOpcode() == OPC) {
625 // NOLINTNEXTLINE(readability-magic-numbers)
626 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Remove consecutive " << GetOpcodeString(OPC);
627 ReplaceUsersAndRemoveCheck(userInst, inst);
628 // Start iteration from beginning, because the new successors may be added.
629 user = inst->GetUsers().begin();
630 end = inst->GetUsers().end();
631 } else {
632 ++user;
633 }
634 }
635 }
636
637 template <Opcode OPC>
TryRemoveCheckByBounds(Inst * inst,Inst * input)638 bool ChecksElimination::TryRemoveCheckByBounds(Inst *inst, Inst *input)
639 {
640 // NOLINTNEXTLINE(readability-magic-numbers)
641 static_assert(OPC == Opcode::ZeroCheck || OPC == Opcode::NegativeCheck || OPC == Opcode::NotPositiveCheck ||
642 OPC == Opcode::NullCheck);
643 ASSERT(inst->GetOpcode() == OPC);
644 if (input->GetType() == DataType::UINT64) {
645 return false;
646 }
647
648 auto block = inst->GetBasicBlock();
649 auto bri = block->GetGraph()->GetBoundsRangeInfo();
650 auto range = bri->FindBoundsRange(block, input);
651 bool result = false;
652 // NOLINTNEXTLINE(readability-magic-numbers, readability-braces-around-statements, bugprone-branch-clone)
653 if constexpr (OPC == Opcode::ZeroCheck) {
654 result = range.IsLess(BoundsRange(0)) || range.IsMore(BoundsRange(0));
655 } else if constexpr (OPC == Opcode::NullCheck) { // NOLINT
656 result = range.IsMore(BoundsRange(0));
657 } else if constexpr (OPC == Opcode::NegativeCheck) { // NOLINT
658 result = range.IsNotNegative();
659 } else if constexpr (OPC == Opcode::NotPositiveCheck) { // NOLINT
660 result = range.IsPositive();
661 }
662 if (result) {
663 // NOLINTNEXTLINE(readability-magic-numbers)
664 COMPILER_LOG(DEBUG, CHECKS_ELIM) << GetOpcodeString(OPC) << " have correct bounds";
665 ReplaceUsersAndRemoveCheck(inst, input);
666 } else {
667 // NOLINTNEXTLINE(readability-magic-numbers, readability-braces-around-statements)
668 if constexpr (OPC == Opcode::ZeroCheck || OPC == Opcode::NullCheck) {
669 result = range.IsEqual(BoundsRange(0));
670 } else if constexpr (OPC == Opcode::NegativeCheck) { // NOLINT
671 result = range.IsNegative();
672 } else if constexpr (OPC == Opcode::NotPositiveCheck) { // NOLINT
673 result = range.IsNotPositive();
674 }
675 if (result) {
676 COMPILER_LOG(DEBUG, CHECKS_ELIM)
677 // NOLINTNEXTLINE(readability-magic-numbers)
678 << GetOpcodeString(OPC) << " must throw, saved for replace by unconditional deoptimize";
679 PushNewCheckMustThrow(inst);
680 }
681 }
682 return result;
683 }
684
685 template <Opcode OPC, bool CHECK_FULL_DOM>
TryRemoveCheck(Inst * inst)686 bool ChecksElimination::TryRemoveCheck(Inst *inst)
687 {
688 // NOLINTNEXTLINE(readability-magic-numbers)
689 static_assert(OPC == Opcode::ZeroCheck || OPC == Opcode::NegativeCheck || OPC == Opcode::NotPositiveCheck ||
690 OPC == Opcode::NullCheck);
691 ASSERT(inst->GetOpcode() == OPC);
692
693 // NOLINTNEXTLINE(readability-magic-numbers)
694 TryRemoveDominatedChecks<OPC, CHECK_FULL_DOM>(inst);
695 // NOLINTNEXTLINE(readability-magic-numbers)
696 TryRemoveConsecutiveChecks<OPC>(inst);
697
698 auto input = inst->GetInput(0).GetInst();
699 // NOLINTNEXTLINE(readability-magic-numbers)
700 return TryRemoveCheckByBounds<OPC>(inst, input);
701 }
702
703 template <Opcode OPC>
TryOptimizeOverflowCheck(Inst * inst)704 void ChecksElimination::TryOptimizeOverflowCheck(Inst *inst)
705 {
706 auto block = inst->GetBasicBlock();
707 auto bri = block->GetGraph()->GetBoundsRangeInfo();
708 auto range = bri->FindBoundsRange(block, inst);
709 bool canOverflow = true;
710 if constexpr (OPC == Opcode::NegOverflowAndZeroCheck) {
711 canOverflow = range.CanOverflowNeg(DataType::INT32);
712 } else {
713 canOverflow = range.CanOverflow(DataType::INT32);
714 }
715 if (!canOverflow) {
716 block->RemoveOverflowCheck(inst);
717 SetApplied();
718 return;
719 }
720 bool constInputs = true;
721 for (size_t i = 0; i < inst->GetInputsCount() - 1; ++i) {
722 constInputs &= inst->GetInput(i).GetInst()->IsConst();
723 }
724 if (constInputs) {
725 // replace by deopt
726 PushNewCheckMustThrow(inst);
727 return;
728 }
729 PushNewCheckForMoveOutOfLoop(inst);
730 }
731
FindSaveState(Loop * loop)732 Inst *ChecksElimination::FindSaveState(Loop *loop)
733 {
734 auto block = loop->GetPreHeader();
735 while (block != nullptr) {
736 for (const auto &inst : block->InstsSafeReverse()) {
737 if (inst->GetOpcode() == Opcode::SaveStateDeoptimize || inst->GetOpcode() == Opcode::SaveState) {
738 return inst;
739 }
740 }
741 auto next = block->GetDominator();
742 // The case when the dominant block is the head of a inner loop
743 if (next != nullptr && next->GetLoop()->GetOuterLoop() == block->GetLoop()) {
744 return nullptr;
745 }
746 block = next;
747 }
748 return nullptr;
749 }
750
FindOptimalSaveStateForHoist(Inst * inst,Inst ** optimalInsertAfter)751 Inst *ChecksElimination::FindOptimalSaveStateForHoist(Inst *inst, Inst **optimalInsertAfter)
752 {
753 ASSERT(inst->RequireState());
754 auto block = inst->GetBasicBlock();
755 if (block == nullptr) {
756 return nullptr;
757 }
758 auto loop = block->GetLoop();
759 *optimalInsertAfter = nullptr;
760
761 while (!loop->IsRoot() && !loop->GetHeader()->IsOsrEntry() && !loop->IsIrreducible()) {
762 for (auto backEdge : loop->GetBackEdges()) {
763 if (!block->IsDominate(backEdge)) {
764 // avoid taking checks out of slowpath
765 return *optimalInsertAfter;
766 }
767 }
768 // Find save state
769 Inst *ss = FindSaveState(loop);
770 if (ss == nullptr) {
771 return *optimalInsertAfter;
772 }
773 auto insertAfter = ss;
774
775 // Check that inputs are dominate on ss
776 bool inputsAreDominate = true;
777 for (size_t i = 0; i < inst->GetInputsCount() - 1; ++i) {
778 auto input = inst->GetInput(i).GetInst();
779 if (input->IsDominate(insertAfter)) {
780 continue;
781 }
782 if (insertAfter->GetBasicBlock() == input->GetBasicBlock()) {
783 insertAfter = input;
784 } else {
785 inputsAreDominate = false;
786 break;
787 }
788 }
789
790 if (!inputsAreDominate) {
791 return *optimalInsertAfter;
792 }
793 *optimalInsertAfter = insertAfter;
794 if (insertAfter != ss) {
795 // some inputs are dominate on insert_after but not dominate on ss, stop here
796 // the only case when return value is not equal to *optimal_insert_after
797 return ss;
798 }
799 block = loop->GetHeader(); // block will be used to check for hot path
800 loop = loop->GetOuterLoop();
801 }
802 return *optimalInsertAfter;
803 }
804
InsertInstAfter(Inst * inst,Inst * after,BasicBlock * block)805 void ChecksElimination::InsertInstAfter(Inst *inst, Inst *after, BasicBlock *block)
806 {
807 if (after->IsPhi()) {
808 block->PrependInst(inst);
809 } else {
810 block->InsertAfter(inst, after);
811 }
812 }
813
InsertBoundsCheckDeoptimization(ConditionCode cc,Inst * left,int64_t val,Inst * right,Inst * ss,Inst * insertAfter,Opcode newLeftOpcode)814 void ChecksElimination::InsertBoundsCheckDeoptimization(ConditionCode cc, Inst *left, int64_t val, Inst *right,
815 Inst *ss, Inst *insertAfter, Opcode newLeftOpcode)
816 {
817 auto block = insertAfter->GetBasicBlock();
818 Inst *newLeft = nullptr;
819 if (val == 0) {
820 newLeft = left;
821 } else if (left == nullptr) {
822 ASSERT(newLeftOpcode == Opcode::Add);
823 newLeft = GetGraph()->FindOrCreateConstant(val);
824 } else {
825 auto cnst = GetGraph()->FindOrCreateConstant(val);
826 newLeft = GetGraph()->CreateInst(newLeftOpcode);
827 newLeft->SetType(DataType::INT32);
828 newLeft->SetInput(0, left);
829 newLeft->SetInput(1, cnst);
830 if (newLeft->RequireState()) {
831 newLeft->SetSaveState(ss);
832 }
833 InsertInstAfter(newLeft, insertAfter, block);
834 insertAfter = newLeft;
835 }
836 auto deoptComp = GetGraph()->CreateInstCompare(DataType::BOOL, INVALID_PC, newLeft, right, DataType::INT32, cc);
837 auto deopt =
838 GetGraph()->CreateInstDeoptimizeIf(DataType::NO_TYPE, ss->GetPc(), deoptComp, ss, DeoptimizeType::BOUNDS_CHECK);
839 InsertInstAfter(deoptComp, insertAfter, block);
840 block->InsertAfter(deopt, deoptComp);
841 }
842
InsertDeoptimization(ConditionCode cc,Inst * left,Inst * right,Inst * ss,Inst * insertAfter,DeoptimizeType deoptType)843 Inst *ChecksElimination::InsertDeoptimization(ConditionCode cc, Inst *left, Inst *right, Inst *ss, Inst *insertAfter,
844 DeoptimizeType deoptType)
845 {
846 auto deoptComp = GetGraph()->CreateInstCompare(DataType::BOOL, INVALID_PC, left, right, left->GetType(), cc);
847 auto deopt = GetGraph()->CreateInstDeoptimizeIf(DataType::NO_TYPE, ss->GetPc(), deoptComp, ss, deoptType);
848 auto block = insertAfter->GetBasicBlock();
849 block->InsertAfter(deoptComp, insertAfter);
850 block->InsertAfter(deopt, deoptComp);
851 return deopt;
852 }
853
FindLoopInfo(Loop * loop)854 std::optional<LoopInfo> ChecksElimination::FindLoopInfo(Loop *loop)
855 {
856 Inst *ss = FindSaveState(loop);
857 if (ss == nullptr) {
858 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "SaveState isn't founded";
859 return std::nullopt;
860 }
861 auto loopParser = CountableLoopParser(*loop);
862 if (auto loopInfo = loopParser.Parse()) {
863 auto loopInfoValue = loopInfo.value();
864 if (loopInfoValue.normalizedCc == CC_NE) {
865 return std::nullopt;
866 }
867 bool isHeadLoopExit = loopInfoValue.ifImm->GetBasicBlock() == loop->GetHeader();
868 bool hasPreHeaderCompare = CountableLoopParser::HasPreHeaderCompare(loop, loopInfoValue);
869 ASSERT(loopInfoValue.index->GetOpcode() == Opcode::Phi);
870 if (loopInfoValue.isInc) {
871 return std::make_tuple(loopInfoValue, ss, loopInfoValue.init, loopInfoValue.test,
872 loopInfoValue.normalizedCc == CC_LE ? CC_LE : CC_LT, isHeadLoopExit,
873 hasPreHeaderCompare);
874 }
875 return std::make_tuple(loopInfoValue, ss, loopInfoValue.test, loopInfoValue.init, CC_LE, isHeadLoopExit,
876 hasPreHeaderCompare);
877 }
878 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Not countable loop isn't supported";
879 return std::nullopt;
880 }
881
InsertNewLenArray(Inst * lenArray,Inst * ss)882 Inst *ChecksElimination::InsertNewLenArray(Inst *lenArray, Inst *ss)
883 {
884 if (lenArray->IsDominate(ss)) {
885 return lenArray;
886 }
887 if (lenArray->GetOpcode() == Opcode::LenArray) {
888 auto nullCheck = lenArray->GetInput(0).GetInst();
889 auto ref = lenArray->GetDataFlowInput(nullCheck);
890 if (ref->IsDominate(ss)) {
891 // Build nullcheck + lenarray before loop
892 auto nullcheck = GetGraph()->CreateInstNullCheck(DataType::REFERENCE, ss->GetPc(), ref, ss);
893 nullcheck->SetFlag(inst_flags::CAN_DEOPTIMIZE);
894 auto newLenArray = lenArray->Clone(GetGraph());
895 newLenArray->SetInput(0, nullcheck);
896 auto block = ss->GetBasicBlock();
897 block->InsertAfter(newLenArray, ss);
898 block->InsertAfter(nullcheck, ss);
899 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Builded new NullCheck(id=" << nullcheck->GetId()
900 << ") and LenArray(id=" << newLenArray->GetId() << ") before loop";
901 ChecksElimination::VisitNullCheck<true>(this, nullcheck);
902 return newLenArray;
903 }
904 }
905 return nullptr;
906 }
907
InsertDeoptimizationForIndexOverflow(CountableLoopInfo * countableLoopInfo,BoundsRange indexUpperRange,Inst * ss)908 void ChecksElimination::InsertDeoptimizationForIndexOverflow(CountableLoopInfo *countableLoopInfo,
909 BoundsRange indexUpperRange, Inst *ss)
910 {
911 auto loopCc = countableLoopInfo->normalizedCc;
912 if (loopCc == CC_LT || loopCc == CC_LE) {
913 auto loopUpper = countableLoopInfo->test;
914 auto step = countableLoopInfo->constStep;
915 auto indexType = countableLoopInfo->index->GetType();
916 ASSERT(indexType == DataType::INT32);
917 auto maxUpper = BoundsRange::GetMax(indexType) - step + (loopCc == CC_LT ? 1 : 0);
918 auto bri = loopUpper->GetBasicBlock()->GetGraph()->GetBoundsRangeInfo();
919 auto loopUpperRange = bri->FindBoundsRange(countableLoopInfo->index->GetBasicBlock(), loopUpper);
920 // Upper bound of loop index assuming (index + max_add < len_array)
921 indexUpperRange = indexUpperRange.Add(BoundsRange(step)).FitInType(indexType);
922 if (!BoundsRange(maxUpper).IsMoreOrEqual(loopUpperRange) && indexUpperRange.IsMaxRange(indexType)) {
923 // loop index can overflow
924 Inst *insertAfter = loopUpper->IsDominate(ss) ? ss : loopUpper;
925 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Build deoptimize for loop index overflow";
926 // Create deoptimize if loop index can become negative
927 InsertBoundsCheckDeoptimization(ConditionCode::CC_LT, nullptr, maxUpper, loopUpper, ss, insertAfter);
928 }
929 }
930 }
931
NeedUpperDeoptimization(BasicBlock * header,Inst * lenArray,BoundsRange lenArrayRange,Inst * upper,BoundsRange upperRange,int64_t maxAdd,ConditionCode cc,bool * insertNewLenArray)932 bool ChecksElimination::NeedUpperDeoptimization(BasicBlock *header, Inst *lenArray, BoundsRange lenArrayRange,
933 Inst *upper, BoundsRange upperRange, int64_t maxAdd, ConditionCode cc,
934 bool *insertNewLenArray)
935 {
936 if (maxAdd == std::numeric_limits<int64_t>::min()) {
937 return false;
938 }
939 if (upperRange.Add(BoundsRange(maxAdd)).IsLess(lenArrayRange)) {
940 return false;
941 }
942 auto bri = GetGraph()->GetBoundsRangeInfo();
943 auto newUpper = upper;
944 auto newMaxAdd = maxAdd;
945 int64_t upperAdd = 0;
946 if (IsInstIncOrDec(upper)) {
947 upperAdd = GetInc(upper);
948 newMaxAdd += upperAdd;
949 newUpper = upper->GetInput(0).GetInst();
950 }
951 if (lenArray == newUpper) {
952 if (newMaxAdd < 0 || (newMaxAdd == 0 && cc == CC_LT)) {
953 return false;
954 }
955 }
956 auto useUpperLen = upperAdd >= 0 || !upperRange.IsMaxRange(upper->GetType());
957 if (useUpperLen && newMaxAdd <= 0) {
958 auto newUpperRange = bri->FindBoundsRange(header, newUpper);
959 if (newUpperRange.GetLenArray() == lenArray) {
960 return false;
961 }
962 }
963 *insertNewLenArray = newUpper != lenArray;
964 return true;
965 }
966
TryInsertDeoptimizationForLargeStep(ConditionCode cc,Inst * resultLenArray,Inst * lower,Inst * upper,int64_t maxAdd,Inst * insertDeoptAfter,Inst * ss,uint64_t constStep)967 bool ChecksElimination::TryInsertDeoptimizationForLargeStep(ConditionCode cc, Inst *resultLenArray, Inst *lower,
968 Inst *upper, int64_t maxAdd, Inst *insertDeoptAfter,
969 Inst *ss, uint64_t constStep)
970 {
971 auto block = insertDeoptAfter->GetBasicBlock();
972 if (!lower->IsDominate(insertDeoptAfter)) {
973 if (lower->GetBasicBlock() == block) {
974 insertDeoptAfter = lower;
975 } else {
976 return false;
977 }
978 }
979 auto subValue = lower;
980 if (cc == CC_LT) {
981 subValue = GetGraph()->CreateInstAdd(DataType::INT32, INVALID_PC, lower, GetGraph()->FindOrCreateConstant(1));
982 InsertInstAfter(subValue, insertDeoptAfter, block);
983 insertDeoptAfter = subValue;
984 }
985 auto sub = GetGraph()->CreateInstSub(DataType::INT32, INVALID_PC, upper, subValue);
986 InsertInstAfter(sub, insertDeoptAfter, block);
987 auto mod = GetGraph()->CreateInstMod(DataType::INT32, INVALID_PC, sub, GetGraph()->FindOrCreateConstant(constStep));
988 block->InsertAfter(mod, sub);
989 if (resultLenArray == upper) {
990 auto maxAddConst = GetGraph()->FindOrCreateConstant(maxAdd);
991 // (upper - lower [- 1]) % step </<= max_add
992 InsertBoundsCheckDeoptimization(cc, mod, 0, maxAddConst, ss, mod, Opcode::NOP);
993 } else {
994 // result_len_array - max_add </<= upper - (upper - lower [- 1]) % step
995 auto maxIndexValue = GetGraph()->CreateInstSub(DataType::INT32, INVALID_PC, upper, mod);
996 block->InsertAfter(maxIndexValue, mod);
997 auto opcode = maxAdd > 0 ? Opcode::Sub : Opcode::SubOverflowCheck;
998 InsertBoundsCheckDeoptimization(cc, resultLenArray, maxAdd, maxIndexValue, ss, maxIndexValue, opcode);
999 }
1000 return true;
1001 }
1002
TryInsertDeoptimization(LoopInfo loopInfo,Inst * lenArray,int64_t maxAdd,int64_t minAdd,bool hasCheckInHeader)1003 bool ChecksElimination::TryInsertDeoptimization(LoopInfo loopInfo, Inst *lenArray, int64_t maxAdd, int64_t minAdd,
1004 bool hasCheckInHeader)
1005 {
1006 auto [countable_loop_info, ss, lower, upper, cc, is_head_loop_exit, has_pre_header_compare] = loopInfo;
1007 ASSERT(cc == CC_LT || cc == CC_LE);
1008 auto bri = GetGraph()->GetBoundsRangeInfo();
1009 auto header = countable_loop_info.index->GetBasicBlock();
1010 auto upperRange = bri->FindBoundsRange(header, upper);
1011 auto lowerRange = bri->FindBoundsRange(header, lower);
1012 auto lenArrayRange = bri->FindBoundsRange(header, lenArray);
1013 auto hasCheckBeforeExit = hasCheckInHeader || !is_head_loop_exit;
1014 if (!has_pre_header_compare && !lowerRange.IsLess(upperRange) && hasCheckBeforeExit) {
1015 // if lower > upper, removing BoundsCheck may be wrong for the first iteration
1016 return false;
1017 }
1018 uint64_t lowerInc = (countable_loop_info.normalizedCc == CC_GT ? 1 : 0);
1019 bool needLowerDeopt = (minAdd != std::numeric_limits<int64_t>::max()) &&
1020 !lowerRange.Add(BoundsRange(minAdd)).Add(BoundsRange(lowerInc)).IsNotNegative();
1021 bool insertLowerDeopt = lower->IsDominate(ss);
1022 if (needLowerDeopt && !insertLowerDeopt) {
1023 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Unable to build deoptimize for lower value";
1024 return false;
1025 }
1026
1027 bool insertNewLenArray;
1028 if (NeedUpperDeoptimization(header, lenArray, lenArrayRange, upper, upperRange, maxAdd, cc, &insertNewLenArray)) {
1029 auto resultLenArray = insertNewLenArray ? InsertNewLenArray(lenArray, ss) : lenArray;
1030 if (resultLenArray == nullptr) {
1031 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Unable to build deoptimize for upper value";
1032 return false;
1033 }
1034 auto insertDeoptAfter = lenArray != resultLenArray ? resultLenArray : ss;
1035 if (!upper->IsDominate(insertDeoptAfter)) {
1036 insertDeoptAfter = upper;
1037 }
1038 ASSERT(insertDeoptAfter->GetBasicBlock()->IsDominate(header));
1039 if (insertDeoptAfter->GetBasicBlock() == header) {
1040 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Unable to build deoptimize for upper value";
1041 return false;
1042 }
1043 auto constStep = countable_loop_info.constStep;
1044 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Try to build deoptimize for upper value";
1045 if (constStep == 1 ||
1046 (countable_loop_info.normalizedCc == CC_GT || countable_loop_info.normalizedCc == CC_GE)) {
1047 auto opcode = maxAdd > 0 ? Opcode::Sub : Opcode::SubOverflowCheck;
1048 // Create deoptimize if result_len_array - max_add <=(<) upper
1049 // result_len_array is >= 0, so if max_add > 0, overflow is not possible
1050 // that's why we do not add max_add to upper instead
1051 InsertBoundsCheckDeoptimization(cc, resultLenArray, maxAdd, upper, ss, insertDeoptAfter, opcode);
1052 } else if (lowerRange.IsConst() && lowerRange.GetLeft() == 0 && countable_loop_info.normalizedCc == CC_LT &&
1053 resultLenArray == upper && maxAdd == static_cast<int64_t>(constStep) - 1) {
1054 // For (int i = 0; i < len; i += x) process(a[i], ..., a[i + x - 1])
1055 // deoptimize if len % x != 0
1056 auto zeroConst = GetGraph()->FindOrCreateConstant(0);
1057 InsertBoundsCheckDeoptimization(ConditionCode::CC_NE, resultLenArray, constStep, zeroConst, ss,
1058 insertDeoptAfter, Opcode::Mod);
1059 } else if (!TryInsertDeoptimizationForLargeStep(cc, resultLenArray, lower, upper, maxAdd, insertDeoptAfter, ss,
1060 constStep)) {
1061 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Unable to build deoptimize for upper value with step > 1";
1062 return false;
1063 }
1064 }
1065 InsertDeoptimizationForIndexOverflow(&countable_loop_info, lenArrayRange.Sub(BoundsRange(maxAdd)), ss);
1066 if (needLowerDeopt) {
1067 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Build deoptimize for lower value";
1068 // Create deoptimize if lower < 0 (or -1 for loop with CC_GT)
1069 auto lowerConst = GetGraph()->FindOrCreateConstant(-lowerInc);
1070 InsertBoundsCheckDeoptimization(ConditionCode::CC_LT, lower, minAdd, lowerConst, ss, ss);
1071 }
1072 return true;
1073 }
1074
HoistLoopInvariantBoundsChecks(Inst * lenArray,GroupedBoundsChecks * indexBoundschecks,Loop * loop)1075 void ChecksElimination::HoistLoopInvariantBoundsChecks(Inst *lenArray, GroupedBoundsChecks *indexBoundschecks,
1076 Loop *loop)
1077 {
1078 auto lenArrLoop = lenArray->GetBasicBlock()->GetLoop();
1079 // len_array isn't loop invariant
1080 if (lenArrLoop == loop) {
1081 return;
1082 }
1083 for (auto &[index, bound_checks_info] : *indexBoundschecks) {
1084 // Check that index is loop invariant, if index is nullptr it means that it was a constant
1085 if (index != nullptr && index->GetBasicBlock()->GetLoop() == loop) {
1086 continue;
1087 }
1088 for (auto boundsCheck : std::get<0>(bound_checks_info)) {
1089 PushNewCheckForMoveOutOfLoop(boundsCheck);
1090 }
1091 }
1092 }
1093
ProcessingGroupBoundsCheck(GroupedBoundsChecks * indexBoundschecks,LoopInfo loopInfo,Inst * lenArray)1094 void ChecksElimination::ProcessingGroupBoundsCheck(GroupedBoundsChecks *indexBoundschecks, LoopInfo loopInfo,
1095 Inst *lenArray)
1096 {
1097 auto phiIndex = std::get<0>(loopInfo).index;
1098 if (indexBoundschecks->find(phiIndex) == indexBoundschecks->end()) {
1099 HoistLoopInvariantBoundsChecks(lenArray, indexBoundschecks, phiIndex->GetBasicBlock()->GetLoop());
1100 return;
1101 }
1102 auto &[insts_to_delete, max_add, min_add] = indexBoundschecks->at(phiIndex);
1103 ASSERT(!insts_to_delete.empty());
1104 bool hasCheckInHeader = false;
1105 for (const auto &inst : insts_to_delete) {
1106 if (inst->GetBasicBlock() == phiIndex->GetBasicBlock()) {
1107 hasCheckInHeader = true;
1108 }
1109 }
1110 if (TryInsertDeoptimization(loopInfo, lenArray, max_add, min_add, hasCheckInHeader)) {
1111 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Delete group of BoundsChecks";
1112 // Delete bounds checks instructions
1113 for (const auto &inst : insts_to_delete) {
1114 ReplaceUsersAndRemoveCheck(inst, inst->GetInput(1).GetInst());
1115 if (indexBoundschecks->find(inst->GetInput(1).GetInst()) != indexBoundschecks->end()) {
1116 indexBoundschecks->erase(inst->GetInput(1).GetInst());
1117 }
1118 }
1119 }
1120 }
1121
ProcessingLoop(Loop * loop,ArenaUnorderedMap<Inst *,GroupedBoundsChecks> * lenarrIndexChecks)1122 void ChecksElimination::ProcessingLoop(Loop *loop, ArenaUnorderedMap<Inst *, GroupedBoundsChecks> *lenarrIndexChecks)
1123 {
1124 auto loopInfo = FindLoopInfo(loop);
1125 if (loopInfo == std::nullopt) {
1126 return;
1127 }
1128 for (auto &[len_array, index_boundschecks] : *lenarrIndexChecks) {
1129 ProcessingGroupBoundsCheck(&index_boundschecks, loopInfo.value(), len_array);
1130 }
1131 }
1132
ReplaceBoundsCheckToDeoptimizationBeforeLoop()1133 void ChecksElimination::ReplaceBoundsCheckToDeoptimizationBeforeLoop()
1134 {
1135 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Start ReplaceBoundsCheckToDeoptimizationBeforeLoop";
1136 for (auto &[loop, lenarr_index_checks] : boundsChecks_) {
1137 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Processing loop with id = " << loop->GetId();
1138 if (loop->IsRoot()) {
1139 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Skip root loop";
1140 continue;
1141 }
1142 ProcessingLoop(loop, &lenarr_index_checks);
1143 }
1144 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Finish ReplaceBoundsCheckToDeoptimizationBeforeLoop";
1145 }
1146
MoveCheckOutOfLoop()1147 void ChecksElimination::MoveCheckOutOfLoop()
1148 {
1149 for (auto inst : checksForMoveOutOfLoop_) {
1150 Inst *insertAfter = nullptr;
1151 auto ss = FindOptimalSaveStateForHoist(inst, &insertAfter);
1152 if (ss == nullptr) {
1153 continue;
1154 }
1155 ASSERT(insertAfter != nullptr);
1156 ASSERT(ss->GetBasicBlock() == insertAfter->GetBasicBlock());
1157 auto block = inst->GetBasicBlock();
1158 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Move check " << GetOpcodeString(inst->GetOpcode())
1159 << " with id = " << inst->GetId() << " from bb " << block->GetId() << " to bb "
1160 << ss->GetBasicBlock()->GetId();
1161 block->EraseInst(inst);
1162 ss->GetBasicBlock()->InsertAfter(inst, insertAfter);
1163 inst->SetSaveState(ss);
1164 inst->SetPc(ss->GetPc());
1165 inst->SetFlag(inst_flags::CAN_DEOPTIMIZE);
1166 SetApplied();
1167 }
1168 }
1169
FindSaveState(const InstVector & instsToDelete)1170 Inst *ChecksElimination::FindSaveState(const InstVector &instsToDelete)
1171 {
1172 for (auto boundsCheck : instsToDelete) {
1173 bool isDominate = true;
1174 for (auto boundsCheckTest : instsToDelete) {
1175 if (boundsCheck == boundsCheckTest) {
1176 continue;
1177 }
1178 isDominate &= boundsCheck->IsDominate(boundsCheckTest);
1179 }
1180 if (isDominate) {
1181 constexpr auto IMM_2 = 2;
1182 return boundsCheck->GetInput(IMM_2).GetInst();
1183 }
1184 }
1185 return nullptr;
1186 }
1187
ReplaceBoundsCheckToDeoptimizationInLoop()1188 void ChecksElimination::ReplaceBoundsCheckToDeoptimizationInLoop()
1189 {
1190 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Start ReplaceBoundsCheckToDeoptimizationInLoop";
1191 for (auto &item : boundsChecks_) {
1192 for (auto &[len_array, index_boundschecks_map] : item.second) {
1193 for (auto &[index, it] : index_boundschecks_map) {
1194 auto [insts_to_delete, max, min] = it;
1195 constexpr auto MIN_BOUNDSCHECKS_NUM = 2;
1196 if (insts_to_delete.size() <= MIN_BOUNDSCHECKS_NUM) {
1197 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Skip small group of BoundsChecks";
1198 continue;
1199 }
1200 // Try to replace more than 2 bounds checks to deoptimization in loop
1201 auto saveState = FindSaveState(insts_to_delete);
1202 if (saveState == nullptr) {
1203 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "SaveState isn't founded";
1204 continue;
1205 }
1206 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Replace group of BoundsChecks on deoptimization in loop";
1207 auto insertAfter = len_array->IsDominate(saveState) ? saveState : len_array;
1208 if (max != std::numeric_limits<int64_t>::min()) {
1209 // Create deoptimize if max_index >= len_array
1210 InsertBoundsCheckDeoptimization(ConditionCode::CC_GE, index, max, len_array, saveState,
1211 insertAfter);
1212 }
1213 if (index != nullptr && min != std::numeric_limits<int64_t>::max()) {
1214 // Create deoptimize if min_index < 0
1215 auto zeroConst = GetGraph()->FindOrCreateConstant(0);
1216 InsertBoundsCheckDeoptimization(ConditionCode::CC_LT, index, min, zeroConst, saveState,
1217 insertAfter);
1218 } else {
1219 // No lower check needed based on BoundsAnalysis
1220 // if index is null, group of bounds checks consists of constants
1221 ASSERT(min >= 0);
1222 }
1223 for (const auto &inst : insts_to_delete) {
1224 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Delete group of BoundsChecks";
1225 ReplaceUsersAndRemoveCheck(inst, inst->GetInput(1).GetInst());
1226 }
1227 }
1228 }
1229 }
1230 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Finish ReplaceBoundsCheckToDeoptimizationInLoop";
1231 }
1232
ReplaceCheckMustThrowByUnconditionalDeoptimize()1233 void ChecksElimination::ReplaceCheckMustThrowByUnconditionalDeoptimize()
1234 {
1235 for (auto &inst : checksMustThrow_) {
1236 auto block = inst->GetBasicBlock();
1237 if (block != nullptr) {
1238 COMPILER_LOG(DEBUG, CHECKS_ELIM)
1239 << "Replace check with id = " << inst->GetId() << " by uncondition deoptimize";
1240 block->ReplaceInstByDeoptimize(inst);
1241 SetApplied();
1242 SetLoopDeleted();
1243 }
1244 }
1245 }
1246
1247 } // namespace panda::compiler
1248