1 /*
2 * Copyright (c) 2021-2024 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "compiler_logger.h"
17 #include "checks_elimination.h"
18 #include "optimizer/analysis/alias_analysis.h"
19 #include "optimizer/analysis/bounds_analysis.h"
20 #include "optimizer/analysis/dominators_tree.h"
21 #include "optimizer/analysis/loop_analyzer.h"
22 #include "optimizer/analysis/object_type_propagation.h"
23 #include "optimizer/ir/graph_visitor.h"
24 #include "optimizer/ir/analysis.h"
25
26 namespace ark::compiler {
27
RunImpl()28 bool ChecksElimination::RunImpl()
29 {
30 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Start ChecksElimination";
31 GetGraph()->RunPass<DominatorsTree>();
32 GetGraph()->RunPass<LoopAnalyzer>();
33 GetGraph()->RunPass<ObjectTypePropagation>();
34
35 VisitGraph();
36
37 if (g_options.IsCompilerEnableReplacingChecksOnDeoptimization()) {
38 if (!GetGraph()->IsOsrMode()) {
39 ReplaceBoundsCheckToDeoptimizationBeforeLoop();
40 MoveCheckOutOfLoop();
41 }
42 ReplaceBoundsCheckToDeoptimizationInLoop();
43
44 ReplaceCheckMustThrowByUnconditionalDeoptimize();
45 }
46 if (IsLoopDeleted() && GetGraph()->IsOsrMode()) {
47 CleanupGraphSaveStateOSR(GetGraph());
48 }
49 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "ChecksElimination " << (IsApplied() ? "is" : "isn't") << " applied";
50 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Finish ChecksElimination";
51 return isApplied_;
52 }
53
InvalidateAnalyses()54 void ChecksElimination::InvalidateAnalyses()
55 {
56 GetGraph()->InvalidateAnalysis<DominatorsTree>();
57 // Already "LoopAnalyzer" was ran in "CleanupGraphSaveStateOSR"
58 // in case (IsLoopDeleted() && GetGraph()->IsOsrMode())
59 if (!(IsLoopDeleted() && GetGraph()->IsOsrMode())) {
60 GetGraph()->InvalidateAnalysis<LoopAnalyzer>();
61 }
62 GetGraph()->InvalidateAnalysis<BoundsAnalysis>();
63 GetGraph()->InvalidateAnalysis<AliasAnalysis>();
64 }
65
66 template <bool CHECK_FULL_DOM>
VisitNullCheck(GraphVisitor * v,Inst * inst)67 void ChecksElimination::VisitNullCheck(GraphVisitor *v, Inst *inst)
68 {
69 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Start visit NullCheck with id = " << inst->GetId();
70
71 auto ref = inst->GetInput(0).GetInst();
72 static_cast<ChecksElimination *>(v)->TryRemoveDominatedNullChecks(inst, ref);
73
74 if (!static_cast<ChecksElimination *>(v)->TryRemoveCheck<Opcode::NullCheck, CHECK_FULL_DOM>(inst)) {
75 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "NullCheck couldn't be deleted";
76 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "NullCheck saved for further replacing on deoptimization";
77 static_cast<ChecksElimination *>(v)->PushNewCheckForMoveOutOfLoop(inst);
78 }
79 }
80
VisitNegativeCheck(GraphVisitor * v,Inst * inst)81 void ChecksElimination::VisitNegativeCheck(GraphVisitor *v, Inst *inst)
82 {
83 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Start visit NegativeCheck with id = " << inst->GetId();
84 if (!static_cast<ChecksElimination *>(v)->TryRemoveCheck<Opcode::NegativeCheck>(inst)) {
85 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "NegativeCheck couldn't be deleted";
86 static_cast<ChecksElimination *>(v)->PushNewCheckForMoveOutOfLoop(inst);
87 }
88 }
89
VisitNotPositiveCheck(GraphVisitor * v,Inst * inst)90 void ChecksElimination::VisitNotPositiveCheck(GraphVisitor *v, Inst *inst)
91 {
92 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Start visit NotPositiveCheck with id = " << inst->GetId();
93 if (!static_cast<ChecksElimination *>(v)->TryRemoveCheck<Opcode::NotPositiveCheck>(inst)) {
94 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "NotPositiveCheck couldn't be deleted";
95 static_cast<ChecksElimination *>(v)->PushNewCheckForMoveOutOfLoop(inst);
96 }
97 }
98
VisitZeroCheck(GraphVisitor * v,Inst * inst)99 void ChecksElimination::VisitZeroCheck(GraphVisitor *v, Inst *inst)
100 {
101 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Start visit ZeroCheck with id = " << inst->GetId();
102 if (!static_cast<ChecksElimination *>(v)->TryRemoveCheck<Opcode::ZeroCheck>(inst)) {
103 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "ZeroCheck couldn't be deleted";
104 static_cast<ChecksElimination *>(v)->PushNewCheckForMoveOutOfLoop(inst);
105 }
106 }
107
VisitRefTypeCheck(GraphVisitor * v,Inst * inst)108 void ChecksElimination::VisitRefTypeCheck(GraphVisitor *v, Inst *inst)
109 {
110 auto visitor = static_cast<ChecksElimination *>(v);
111 auto storeInst = inst->GetDataFlowInput(inst->GetInput(1).GetInst());
112 // Case: a[i] = nullptr
113 if (storeInst->GetOpcode() == Opcode::NullPtr) {
114 visitor->ReplaceUsersAndRemoveCheck(inst, storeInst);
115 return;
116 }
117 auto arrayInst = inst->GetDataFlowInput(0);
118 auto ref = inst->GetInput(1).GetInst();
119 // Case:
120 // a[1] = obj
121 // a[2] = obj
122 visitor->TryRemoveDominatedChecks<Opcode::RefTypeCheck>(inst, [arrayInst, ref](Inst *userInst) {
123 return userInst->GetDataFlowInput(0) == arrayInst && userInst->GetInput(1) == ref;
124 });
125 visitor->GetGraph()->RunPass<ObjectTypePropagation>();
126 auto arrayTypeInfo = arrayInst->GetObjectTypeInfo();
127 if (arrayTypeInfo && arrayTypeInfo.IsExact()) {
128 auto storeTypeInfo = storeInst->GetObjectTypeInfo();
129 auto arrayClass = arrayTypeInfo.GetClass();
130 auto storeClass = (storeTypeInfo) ? storeTypeInfo.GetClass() : nullptr;
131 if (visitor->GetGraph()->GetRuntime()->CheckStoreArray(arrayClass, storeClass)) {
132 visitor->ReplaceUsersAndRemoveCheck(inst, storeInst);
133 return;
134 }
135 }
136 visitor->PushNewCheckForMoveOutOfLoop(inst);
137 }
138
TryToEliminateAnyTypeCheck(Inst * inst,Inst * instToReplace,AnyBaseType type,AnyBaseType prevType)139 bool ChecksElimination::TryToEliminateAnyTypeCheck(Inst *inst, Inst *instToReplace, AnyBaseType type,
140 AnyBaseType prevType)
141 {
142 auto language = GetGraph()->GetRuntime()->GetMethodSourceLanguage(GetGraph()->GetMethod());
143 auto allowedType = inst->CastToAnyTypeCheck()->GetAllowedInputType();
144 profiling::AnyInputType prevAllowedType;
145 if (instToReplace->GetOpcode() == Opcode::AnyTypeCheck) {
146 prevAllowedType = instToReplace->CastToAnyTypeCheck()->GetAllowedInputType();
147 } else {
148 prevAllowedType = instToReplace->CastToCastValueToAnyType()->GetAllowedInputType();
149 }
150 auto res = IsAnyTypeCanBeSubtypeOf(language, type, prevType, allowedType, prevAllowedType);
151 if (!res) {
152 return false;
153 }
154 if (res.value()) {
155 ReplaceUsersAndRemoveCheck(inst, instToReplace);
156 } else {
157 PushNewCheckMustThrow(inst);
158 }
159 return true;
160 }
161
UpdateHclassChecks(Inst * inst)162 void ChecksElimination::UpdateHclassChecks(Inst *inst)
163 {
164 // AnyTypeCheck HEAP_OBJECT => LoadObject Class => LoadObject Hclass => HclassCheck
165 bool allUsersInlined = false;
166 for (auto &user : inst->GetUsers()) {
167 auto userInst = user.GetInst();
168 if (userInst->IsCall()) {
169 if (userInst->CastToCallDynamic()->IsInlined()) {
170 allUsersInlined = true;
171 } else {
172 return;
173 }
174 } else if (userInst->GetOpcode() == Opcode::LoadObject) {
175 if (userInst->CastToLoadObject()->GetObjectType() == ObjectType::MEM_DYN_CLASS) {
176 continue;
177 }
178 ASSERT(userInst->CastToLoadObject()->GetObjectType() == ObjectType::MEM_DYN_METHOD);
179 if (!IsInlinedCallLoadMethod(userInst)) {
180 return;
181 }
182 allUsersInlined = true;
183 } else {
184 return;
185 }
186 }
187 if (!allUsersInlined) {
188 return;
189 }
190 for (auto &user : inst->GetUsers()) {
191 auto userInst = user.GetInst();
192 auto check = GetHclassCheckFromLoads(userInst);
193 if (!check.has_value()) {
194 continue;
195 }
196 check.value()->CastToHclassCheck()->SetCheckFunctionIsNotClassConstructor(false);
197 SetApplied();
198 }
199 }
200
GetHclassCheckFromLoads(Inst * loadClass)201 std::optional<Inst *> ChecksElimination::GetHclassCheckFromLoads(Inst *loadClass)
202 {
203 if (loadClass->GetOpcode() != Opcode::LoadObject ||
204 loadClass->CastToLoadObject()->GetObjectType() != ObjectType::MEM_DYN_CLASS) {
205 return std::nullopt;
206 }
207 for (auto &usersLoad : loadClass->GetUsers()) {
208 auto loadHclass = usersLoad.GetInst();
209 if (loadHclass->GetOpcode() != Opcode::LoadObject ||
210 loadHclass->CastToLoadObject()->GetObjectType() != ObjectType::MEM_DYN_HCLASS) {
211 continue;
212 }
213 for (auto &usersHclass : loadHclass->GetUsers()) {
214 auto hclassCheck = usersHclass.GetInst();
215 if (hclassCheck->GetOpcode() == Opcode::HclassCheck) {
216 return hclassCheck;
217 }
218 }
219 }
220 return std::nullopt;
221 }
222
IsInlinedCallLoadMethod(Inst * inst)223 bool ChecksElimination::IsInlinedCallLoadMethod(Inst *inst)
224 {
225 bool isMethod = inst->GetOpcode() == Opcode::LoadObject &&
226 inst->CastToLoadObject()->GetObjectType() == ObjectType::MEM_DYN_METHOD;
227 if (!isMethod) {
228 return false;
229 }
230 for (auto &methodUser : inst->GetUsers()) {
231 auto compare = methodUser.GetInst()->CastToCompare();
232 bool isCompare = compare->GetOpcode() == Opcode::Compare && compare->GetCc() == ConditionCode::CC_NE &&
233 compare->GetInputType(0) == DataType::POINTER;
234 if (!isCompare) {
235 return false;
236 }
237 auto deoptimizeIf = compare->GetFirstUser()->GetInst()->CastToDeoptimizeIf();
238 if (deoptimizeIf->GetDeoptimizeType() != DeoptimizeType::INLINE_DYN) {
239 return false;
240 }
241 }
242 return true;
243 }
244
TryRemoveDominatedHclassCheck(Inst * inst)245 void ChecksElimination::TryRemoveDominatedHclassCheck(Inst *inst)
246 {
247 ASSERT(inst->GetOpcode() == Opcode::HclassCheck);
248 // AnyTypeCheck HEAP_OBJECT => LoadObject Class => LoadObject Hclass => HclassCheck
249 auto object = inst->GetInput(0).GetInst()->GetInput(0).GetInst()->GetInput(0).GetInst();
250
251 for (auto &user : object->GetUsers()) {
252 auto userInst = user.GetInst();
253 auto hclassCheck = GetHclassCheckFromLoads(userInst);
254 if (hclassCheck.has_value() && hclassCheck.value() != inst && inst->IsDominate(hclassCheck.value())) {
255 ASSERT(inst->IsDominate(hclassCheck.value()));
256 COMPILER_LOG(DEBUG, CHECKS_ELIM)
257 << GetOpcodeString(Opcode::HclassCheck) << " with id = " << inst->GetId() << " dominate on "
258 << GetOpcodeString(Opcode::HclassCheck) << " with id = " << hclassCheck.value()->GetId();
259 auto block = hclassCheck.value()->GetBasicBlock();
260 auto graph = block->GetGraph();
261 hclassCheck.value()->RemoveInputs();
262 // We don't should update because it's possible to delete one flag in check, when all DynCall is inlined
263 inst->CastToHclassCheck()->ExtendFlags(hclassCheck.value());
264 block->ReplaceInst(hclassCheck.value(), graph->CreateInstNOP());
265 SetApplied();
266 }
267 }
268 }
269
VisitAnyTypeCheck(GraphVisitor * v,Inst * inst)270 void ChecksElimination::VisitAnyTypeCheck(GraphVisitor *v, Inst *inst)
271 {
272 auto visitor = static_cast<ChecksElimination *>(v);
273 auto inputInst = inst->GetInput(0).GetInst();
274 auto type = inst->CastToAnyTypeCheck()->GetAnyType();
275 if (type == AnyBaseType::UNDEFINED_TYPE) {
276 visitor->ReplaceUsersAndRemoveCheck(inst, inputInst);
277 return;
278 }
279 // from:
280 // 2.any CastValueToAnyType ANY_SUBTYPE v1 -> (v4)
281 // 4.any AnyTypeCheck ANY_SUBTYPE v2, v3 -> (....)
282 // to:
283 // 2.any CastValueToAnyType ANY_SUBTYPE v1 -> (...)
284 if (inputInst->GetOpcode() == Opcode::CastValueToAnyType) {
285 visitor->TryToEliminateAnyTypeCheck(inst, inputInst, type, inputInst->CastToCastValueToAnyType()->GetAnyType());
286 return;
287 }
288 // from:
289 // 2.any AnyTypeCheck ANY_SUBTYPE v1, v0 -> (v4)
290 // 4.any AnyTypeCheck ANY_SUBTYPE v2, v3 -> (....)
291 // to:
292 // 2.any AnyTypeCheck ANY_SUBTYPE v1, v0 -> (...)
293 if (inputInst->GetOpcode() == Opcode::AnyTypeCheck) {
294 visitor->TryToEliminateAnyTypeCheck(inst, inputInst, type, inputInst->CastToAnyTypeCheck()->GetAnyType());
295 return;
296 }
297 // from:
298 // 2.any AnyTypeCheck ANY_SUBTYPE v1, v0 -> (v4)
299 // 4.any AnyTypeCheck ANY_SUBTYPE v1, v3 -> (....)
300 // to:
301 // 2.any AnyTypeCheck ANY_SUBTYPE v1, v0 -> (v4,...)
302 bool applied = false;
303 for (auto &user : inputInst->GetUsers()) {
304 auto userInst = user.GetInst();
305 if (userInst == inst) {
306 continue;
307 }
308 if (userInst->GetOpcode() != Opcode::AnyTypeCheck) {
309 continue;
310 }
311 if (!inst->IsDominate(userInst)) {
312 continue;
313 }
314
315 if (visitor->TryToEliminateAnyTypeCheck(userInst, inst, userInst->CastToAnyTypeCheck()->GetAnyType(), type)) {
316 applied = true;
317 }
318 }
319 if (!applied) {
320 visitor->PushNewCheckForMoveOutOfLoop(inst);
321 }
322 // from:
323 // 39.any AnyTypeCheck ECMASCRIPT_HEAP_OBJECT_TYPE v37, v38 -> (v65, v40)
324 // 40.ref LoadObject 4294967288 Class v39 -> (v41)
325 // 41.ref LoadObject 4294967287 Hclass v40
326 // 42. HclassCheck [IsFunc, IsNotClassConstr] v41, v38
327 // ...METHOD IS INLINED ...
328 // to:
329 // 39.any AnyTypeCheck ECMASCRIPT_HEAP_OBJECT_TYPE v37, v38 -> (v65, v40)
330 // 40.ref LoadObject 4294967288 Class v39 -> (v41)
331 // 41.ref LoadObject 4294967287 Hclass v40
332 // 42. HclassCheck [IsFunc] v41, v38
333 // ...METHOD IS INLINED ...
334 if (TryIsDynHeapObject(type)) {
335 visitor->UpdateHclassChecks(inst);
336 }
337 }
338
VisitHclassCheck(GraphVisitor * v,Inst * inst)339 void ChecksElimination::VisitHclassCheck(GraphVisitor *v, Inst *inst)
340 {
341 [[maybe_unused]] auto visitor = static_cast<ChecksElimination *>(v);
342 visitor->TryRemoveDominatedHclassCheck(inst);
343 visitor->PushNewCheckForMoveOutOfLoop(inst);
344 }
345
VisitBoundsCheck(GraphVisitor * v,Inst * inst)346 void ChecksElimination::VisitBoundsCheck(GraphVisitor *v, Inst *inst)
347 {
348 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Start visit BoundsCheck with id = " << inst->GetId();
349 auto block = inst->GetBasicBlock();
350 auto lenArray = inst->GetInput(0).GetInst();
351 auto index = inst->GetInput(1).GetInst();
352 auto visitor = static_cast<ChecksElimination *>(v);
353
354 visitor->TryRemoveDominatedChecks<Opcode::BoundsCheck>(inst, [lenArray, index](Inst *userInst) {
355 return lenArray == userInst->GetInput(0) && index == userInst->GetInput(1);
356 });
357
358 auto bri = block->GetGraph()->GetBoundsRangeInfo();
359 auto lenArrayRange = bri->FindBoundsRange(block, lenArray);
360 auto indexRange = bri->FindBoundsRange(block, index);
361 auto correctUpper = indexRange.IsLess(lenArrayRange) || indexRange.IsLess(lenArray);
362 auto correctLower = indexRange.IsNotNegative();
363 if (correctUpper && correctLower) {
364 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Index of BoundsCheck have correct bounds";
365 visitor->ReplaceUsersAndRemoveCheck(inst, index);
366 return;
367 }
368
369 if (indexRange.IsNegative() || indexRange.IsMoreOrEqual(lenArrayRange)) {
370 COMPILER_LOG(DEBUG, CHECKS_ELIM)
371 << "BoundsCheck have incorrect bounds, saved for replace by unconditional deoptimize";
372 visitor->PushNewCheckMustThrow(inst);
373 return;
374 }
375
376 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "BoundsCheck saved for further replacing on deoptimization";
377 auto loop = GetLoopForBoundsCheck(block, lenArray, index);
378 visitor->PushNewBoundsCheck(loop, lenArray, index, inst, !correctUpper, !correctLower);
379 }
380
VisitCheckCast(GraphVisitor * v,Inst * inst)381 void ChecksElimination::VisitCheckCast(GraphVisitor *v, Inst *inst)
382 {
383 auto visitor = static_cast<ChecksElimination *>(v);
384 visitor->GetGraph()->RunPass<ObjectTypePropagation>();
385 auto result = ObjectTypeCheckElimination::TryEliminateCheckCast(inst);
386 if (result != ObjectTypeCheckElimination::CheckCastEliminateType::INVALID) {
387 visitor->SetApplied();
388 if (result == ObjectTypeCheckElimination::CheckCastEliminateType::MUST_THROW) {
389 visitor->PushNewCheckMustThrow(inst);
390 }
391 return;
392 }
393 if (!inst->CastToCheckCast()->GetOmitNullCheck()) {
394 auto block = inst->GetBasicBlock();
395 auto bri = block->GetGraph()->GetBoundsRangeInfo();
396 auto inputRange = bri->FindBoundsRange(block, inst->GetInput(0).GetInst());
397 if (inputRange.IsMore(BoundsRange(0))) {
398 visitor->SetApplied();
399 inst->CastToCheckCast()->SetOmitNullCheck(true);
400 }
401 }
402 visitor->PushNewCheckForMoveOutOfLoop(inst);
403 }
404
VisitIsInstance(GraphVisitor * v,Inst * inst)405 void ChecksElimination::VisitIsInstance(GraphVisitor *v, Inst *inst)
406 {
407 if (inst->CastToIsInstance()->GetOmitNullCheck()) {
408 return;
409 }
410 auto block = inst->GetBasicBlock();
411 auto bri = block->GetGraph()->GetBoundsRangeInfo();
412 auto inputRange = bri->FindBoundsRange(block, inst->GetInput(0).GetInst());
413 if (inputRange.IsMore(BoundsRange(0))) {
414 auto visitor = static_cast<ChecksElimination *>(v);
415 visitor->SetApplied();
416 inst->CastToIsInstance()->SetOmitNullCheck(true);
417 }
418 }
419
VisitAddOverflowCheck(GraphVisitor * v,Inst * inst)420 void ChecksElimination::VisitAddOverflowCheck(GraphVisitor *v, Inst *inst)
421 {
422 ASSERT(inst->GetType() == DataType::INT32);
423 auto input1 = inst->GetInput(0).GetInst();
424 auto input2 = inst->GetInput(1).GetInst();
425 auto visitor = static_cast<ChecksElimination *>(v);
426 visitor->TryRemoveDominatedChecks<Opcode::AddOverflowCheck>(inst, [input1, input2](Inst *userInst) {
427 return (userInst->GetInput(0) == input1 && userInst->GetInput(1) == input2) ||
428 (userInst->GetInput(0) == input2 && userInst->GetInput(1) == input1);
429 });
430 visitor->TryOptimizeOverflowCheck<Opcode::AddOverflowCheck>(inst);
431 }
432
VisitSubOverflowCheck(GraphVisitor * v,Inst * inst)433 void ChecksElimination::VisitSubOverflowCheck(GraphVisitor *v, Inst *inst)
434 {
435 ASSERT(inst->GetType() == DataType::INT32);
436 auto input1 = inst->GetInput(0).GetInst();
437 auto input2 = inst->GetInput(1).GetInst();
438 auto visitor = static_cast<ChecksElimination *>(v);
439 visitor->TryRemoveDominatedChecks<Opcode::SubOverflowCheck>(inst, [input1, input2](Inst *userInst) {
440 return (userInst->GetInput(0) == input1 && userInst->GetInput(1) == input2);
441 });
442 visitor->TryOptimizeOverflowCheck<Opcode::SubOverflowCheck>(inst);
443 }
444
VisitNegOverflowAndZeroCheck(GraphVisitor * v,Inst * inst)445 void ChecksElimination::VisitNegOverflowAndZeroCheck(GraphVisitor *v, Inst *inst)
446 {
447 ASSERT(inst->GetType() == DataType::INT32);
448 auto input1 = inst->GetInput(0).GetInst();
449 auto visitor = static_cast<ChecksElimination *>(v);
450 visitor->TryRemoveDominatedChecks<Opcode::NegOverflowAndZeroCheck>(
451 inst, [input1](Inst *userInst) { return (userInst->GetInput(0) == input1); });
452 visitor->TryOptimizeOverflowCheck<Opcode::NegOverflowAndZeroCheck>(inst);
453 }
454
ReplaceUsersAndRemoveCheck(Inst * instDel,Inst * instRep)455 void ChecksElimination::ReplaceUsersAndRemoveCheck(Inst *instDel, Inst *instRep)
456 {
457 auto block = instDel->GetBasicBlock();
458 auto graph = block->GetGraph();
459 if (HasOsrEntryBetween(instRep, instDel)) {
460 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Check couldn't be deleted, because in OSR mode we can't replace "
461 "instructions with instructions placed before OSR entry";
462 return;
463 }
464 instDel->ReplaceUsers(instRep);
465 instDel->RemoveInputs();
466 block->ReplaceInst(instDel, graph->CreateInstNOP());
467 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Checks elimination delete " << GetOpcodeString(instDel->GetOpcode())
468 << " with id " << instDel->GetId();
469 graph->GetEventWriter().EventChecksElimination(GetOpcodeString(instDel->GetOpcode()), instDel->GetId(),
470 instDel->GetPc());
471 SetApplied();
472 }
473
IsInstIncOrDec(Inst * inst)474 bool ChecksElimination::IsInstIncOrDec(Inst *inst)
475 {
476 return inst->IsAddSub() && inst->GetInput(1).GetInst()->IsConst();
477 }
478
GetInc(Inst * inst)479 int64_t ChecksElimination::GetInc(Inst *inst)
480 {
481 ASSERT(IsInstIncOrDec(inst));
482 auto val = static_cast<int64_t>(inst->GetInput(1).GetInst()->CastToConstant()->GetIntValue());
483 if (inst->IsSub()) {
484 val = -val;
485 }
486 return val;
487 }
488
GetLoopForBoundsCheck(BasicBlock * block,Inst * lenArray,Inst * index)489 Loop *ChecksElimination::GetLoopForBoundsCheck(BasicBlock *block, Inst *lenArray, Inst *index)
490 {
491 auto parentIndex = IsInstIncOrDec(index) ? index->GetInput(0).GetInst() : index;
492 auto indexBlock = parentIndex->GetBasicBlock();
493 ASSERT(indexBlock != nullptr);
494 auto indexLoop = indexBlock->GetLoop();
495 if (auto loopInfo = CountableLoopParser(*indexLoop).Parse()) {
496 auto input = lenArray;
497 if (lenArray->GetOpcode() == Opcode::LenArray) {
498 // new lenArray can be inserted
499 input = lenArray->GetDataFlowInput(0);
500 }
501 if (loopInfo->index == parentIndex && input->GetBasicBlock()->IsDominate(indexBlock)) {
502 ASSERT(indexBlock == indexLoop->GetHeader());
503 return indexLoop;
504 }
505 }
506 return block->GetLoop();
507 }
508
InitItemForNewIndex(GroupedBoundsChecks * place,Inst * index,Inst * inst,bool checkUpper,bool checkLower)509 void ChecksElimination::InitItemForNewIndex(GroupedBoundsChecks *place, Inst *index, Inst *inst, bool checkUpper,
510 bool checkLower)
511 {
512 ASSERT(inst->GetOpcode() == Opcode::BoundsCheck);
513 InstVector insts(GetGraph()->GetLocalAllocator()->Adapter());
514 insts.push_back(inst);
515 int64_t val = 0;
516 Inst *parentIndex = index;
517 if (IsInstIncOrDec(index)) {
518 val = GetInc(index);
519 parentIndex = index->GetInput(0).GetInst();
520 } else if (index->IsConst()) {
521 val = static_cast<int64_t>(index->CastToConstant()->GetIntValue());
522 parentIndex = nullptr;
523 }
524 auto maxVal = checkUpper ? val : std::numeric_limits<int64_t>::min();
525 auto minVal = checkLower ? val : std::numeric_limits<int64_t>::max();
526 place->emplace_back(std::make_tuple(parentIndex, insts, maxVal, minVal));
527 }
528
PushNewBoundsCheck(Loop * loop,Inst * lenArray,Inst * index,Inst * inst,bool checkUpper,bool checkLower)529 void ChecksElimination::PushNewBoundsCheck(Loop *loop, Inst *lenArray, Inst *index, Inst *inst, bool checkUpper,
530 bool checkLower)
531 {
532 ASSERT(loop != nullptr && lenArray != nullptr && index != nullptr && inst != nullptr);
533 ASSERT(inst->GetOpcode() == Opcode::BoundsCheck);
534 auto loopChecksIt =
535 std::find_if(boundsChecks_.begin(), boundsChecks_.end(), [=](auto p) { return p.first == loop; });
536 if (loopChecksIt == boundsChecks_.end()) {
537 auto &loopLenArrays = boundsChecks_.emplace_back(loop, GetGraph()->GetLocalAllocator()->Adapter());
538 auto &lenArrayIndexes = loopLenArrays.second.emplace_back(lenArray, GetGraph()->GetLocalAllocator()->Adapter());
539 InitItemForNewIndex(&lenArrayIndexes.second, index, inst, checkUpper, checkLower);
540 } else {
541 auto *lenArrays = &loopChecksIt->second;
542 auto lenArrayIt =
543 std::find_if(lenArrays->begin(), lenArrays->end(), [=](auto p) { return p.first == lenArray; });
544 if (lenArrayIt == lenArrays->end()) {
545 auto &lenArrayIndexes = lenArrays->emplace_back(lenArray, GetGraph()->GetLocalAllocator()->Adapter());
546 InitItemForNewIndex(&lenArrayIndexes.second, index, inst, checkUpper, checkLower);
547 } else {
548 auto *indexes = &lenArrayIt->second;
549 PushNewBoundsCheckAtExistingIndexes(indexes, index, inst, checkUpper, checkLower);
550 }
551 }
552 }
553
PushNewBoundsCheckAtExistingIndexes(GroupedBoundsChecks * indexes,Inst * index,Inst * inst,bool checkUpper,bool checkLower)554 void ChecksElimination::PushNewBoundsCheckAtExistingIndexes(GroupedBoundsChecks *indexes, Inst *index, Inst *inst,
555 bool checkUpper, bool checkLower)
556 {
557 auto indexIt = std::find_if(indexes->begin(), indexes->end(), [=](auto p) { return std::get<0>(p) == index; });
558 if (indexIt == indexes->end()) {
559 auto parentIndex = index;
560 int64_t val {};
561 if (IsInstIncOrDec(index)) {
562 parentIndex = index->GetInput(0).GetInst();
563 val = GetInc(index);
564 } else if (index->IsConst()) {
565 parentIndex = nullptr;
566 val = static_cast<int64_t>(index->CastToConstant()->GetIntValue());
567 }
568 auto parentIndexIt =
569 std::find_if(indexes->begin(), indexes->end(), [=](auto p) { return std::get<0>(p) == parentIndex; });
570 if (parentIndex == index || parentIndexIt == indexes->end()) {
571 InitItemForNewIndex(indexes, index, inst, checkUpper, checkLower);
572 } else {
573 auto &boundChecks = std::get<1U>(*parentIndexIt);
574 auto &max = std::get<2U>(*parentIndexIt);
575 auto &min = std::get<3U>(*parentIndexIt);
576 boundChecks.push_back(inst);
577 if (val > max && checkUpper) {
578 max = val;
579 } else if (val < min && checkLower) {
580 min = val;
581 }
582 }
583 } else {
584 auto &boundChecks = std::get<1U>(*indexIt);
585 auto &max = std::get<2U>(*indexIt);
586 auto &min = std::get<3U>(*indexIt);
587 boundChecks.push_back(inst);
588 if (max < 0 && checkUpper) {
589 max = 0;
590 }
591 if (min > 0 && checkLower) {
592 min = 0;
593 }
594 }
595 }
596
TryRemoveDominatedNullChecks(Inst * inst,Inst * ref)597 void ChecksElimination::TryRemoveDominatedNullChecks(Inst *inst, Inst *ref)
598 {
599 for (auto &user : ref->GetUsers()) {
600 auto userInst = user.GetInst();
601 if (((userInst->GetOpcode() == Opcode::IsInstance && !userInst->CastToIsInstance()->GetOmitNullCheck()) ||
602 (userInst->GetOpcode() == Opcode::CheckCast && !userInst->CastToCheckCast()->GetOmitNullCheck())) &&
603 inst->IsDominate(userInst)) {
604 COMPILER_LOG(DEBUG, CHECKS_ELIM)
605 << "NullCheck with id = " << inst->GetId() << " dominate on " << GetOpcodeString(userInst->GetOpcode())
606 << " with id = " << userInst->GetId();
607 if (userInst->GetOpcode() == Opcode::IsInstance) {
608 userInst->CastToIsInstance()->SetOmitNullCheck(true);
609 } else {
610 userInst->CastToCheckCast()->SetOmitNullCheck(true);
611 }
612 SetApplied();
613 }
614 }
615 }
616
617 template <Opcode OPC, bool CHECK_FULL_DOM, typename CheckInputs>
TryRemoveDominatedCheck(Inst * inst,Inst * userInst,CheckInputs checkInputs)618 void ChecksElimination::TryRemoveDominatedCheck(Inst *inst, Inst *userInst, CheckInputs checkInputs)
619 {
620 // NOLINTNEXTLINE(readability-magic-numbers)
621 if (userInst->GetOpcode() == OPC && userInst != inst && userInst->GetType() == inst->GetType() &&
622 checkInputs(userInst) &&
623 (CHECK_FULL_DOM ? inst->IsDominate(userInst) : inst->InSameBlockOrDominate(userInst))) {
624 COMPILER_LOG(DEBUG, CHECKS_ELIM)
625 // NOLINTNEXTLINE(readability-magic-numbers)
626 << GetOpcodeString(OPC) << " with id = " << inst->GetId() << " dominate on " << GetOpcodeString(OPC)
627 << " with id = " << userInst->GetId();
628 ReplaceUsersAndRemoveCheck(userInst, inst);
629 }
630 }
631
632 template <Opcode OPC, bool CHECK_FULL_DOM, typename CheckInputs>
TryRemoveDominatedChecks(Inst * inst,CheckInputs checkInputs)633 void ChecksElimination::TryRemoveDominatedChecks(Inst *inst, CheckInputs checkInputs)
634 {
635 for (auto &directUser : inst->GetInput(0).GetInst()->GetUsers()) {
636 auto directUserInst = directUser.GetInst();
637 if ((OPC != Opcode::NullCheck) && (directUserInst->GetOpcode() == Opcode::NullCheck)) {
638 for (auto &actualUser : directUserInst->GetUsers()) {
639 auto actualUserInst = actualUser.GetInst();
640 TryRemoveDominatedCheck<OPC, CHECK_FULL_DOM>(inst, actualUserInst, checkInputs);
641 }
642 } else {
643 TryRemoveDominatedCheck<OPC, CHECK_FULL_DOM>(inst, directUserInst, checkInputs);
644 }
645 }
646 }
647
648 // Remove consecutive checks: NullCheck -> NullCheck -> NullCheck
649 template <Opcode OPC>
TryRemoveConsecutiveChecks(Inst * inst)650 void ChecksElimination::TryRemoveConsecutiveChecks(Inst *inst)
651 {
652 auto end = inst->GetUsers().end();
653 for (auto user = inst->GetUsers().begin(); user != end;) {
654 auto userInst = (*user).GetInst();
655 // NOLINTNEXTLINE(readability-magic-numbers)
656 if (userInst->GetOpcode() == OPC) {
657 // NOLINTNEXTLINE(readability-magic-numbers)
658 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Remove consecutive " << GetOpcodeString(OPC);
659 ReplaceUsersAndRemoveCheck(userInst, inst);
660 // Start iteration from beginning, because the new successors may be added.
661 user = inst->GetUsers().begin();
662 end = inst->GetUsers().end();
663 } else {
664 ++user;
665 }
666 }
667 }
668
669 template <Opcode OPC>
TryRemoveCheckByBounds(Inst * inst,Inst * input)670 bool ChecksElimination::TryRemoveCheckByBounds(Inst *inst, Inst *input)
671 {
672 // NOLINTNEXTLINE(readability-magic-numbers)
673 static_assert(OPC == Opcode::ZeroCheck || OPC == Opcode::NegativeCheck || OPC == Opcode::NotPositiveCheck ||
674 OPC == Opcode::NullCheck);
675 ASSERT(inst->GetOpcode() == OPC);
676 auto block = inst->GetBasicBlock();
677 auto bri = block->GetGraph()->GetBoundsRangeInfo();
678 auto range = bri->FindBoundsRange(block, input);
679 bool result = false;
680 // NOLINTNEXTLINE(readability-magic-numbers, readability-braces-around-statements, bugprone-branch-clone)
681 if constexpr (OPC == Opcode::ZeroCheck) {
682 result = range.IsLess(BoundsRange(0)) || range.IsMore(BoundsRange(0));
683 } else if constexpr (OPC == Opcode::NullCheck) { // NOLINT
684 result = range.IsMore(BoundsRange(0));
685 } else if constexpr (OPC == Opcode::NegativeCheck) { // NOLINT
686 result = range.IsNotNegative();
687 } else if constexpr (OPC == Opcode::NotPositiveCheck) { // NOLINT
688 result = range.IsPositive();
689 }
690 if (result) {
691 // NOLINTNEXTLINE(readability-magic-numbers)
692 COMPILER_LOG(DEBUG, CHECKS_ELIM) << GetOpcodeString(OPC) << " have correct bounds";
693 ReplaceUsersAndRemoveCheck(inst, input);
694 } else {
695 // NOLINTNEXTLINE(readability-magic-numbers, readability-braces-around-statements)
696 if constexpr (OPC == Opcode::ZeroCheck || OPC == Opcode::NullCheck) {
697 result = range.IsEqual(BoundsRange(0));
698 } else if constexpr (OPC == Opcode::NegativeCheck) { // NOLINT
699 result = range.IsNegative();
700 } else if constexpr (OPC == Opcode::NotPositiveCheck) { // NOLINT
701 result = range.IsNotPositive();
702 }
703 if (result) {
704 COMPILER_LOG(DEBUG, CHECKS_ELIM)
705 // NOLINTNEXTLINE(readability-magic-numbers)
706 << GetOpcodeString(OPC) << " must throw, saved for replace by unconditional deoptimize";
707 PushNewCheckMustThrow(inst);
708 }
709 }
710 return result;
711 }
712
713 template <Opcode OPC, bool CHECK_FULL_DOM>
TryRemoveCheck(Inst * inst)714 bool ChecksElimination::TryRemoveCheck(Inst *inst)
715 {
716 // NOLINTNEXTLINE(readability-magic-numbers)
717 static_assert(OPC == Opcode::ZeroCheck || OPC == Opcode::NegativeCheck || OPC == Opcode::NotPositiveCheck ||
718 OPC == Opcode::NullCheck);
719 ASSERT(inst->GetOpcode() == OPC);
720
721 // NOLINTNEXTLINE(readability-magic-numbers)
722 TryRemoveDominatedChecks<OPC, CHECK_FULL_DOM>(inst);
723 // NOLINTNEXTLINE(readability-magic-numbers)
724 TryRemoveConsecutiveChecks<OPC>(inst);
725
726 auto input = inst->GetInput(0).GetInst();
727 // NOLINTNEXTLINE(readability-magic-numbers)
728 return TryRemoveCheckByBounds<OPC>(inst, input);
729 }
730
731 template <Opcode OPC>
TryOptimizeOverflowCheck(Inst * inst)732 void ChecksElimination::TryOptimizeOverflowCheck(Inst *inst)
733 {
734 auto block = inst->GetBasicBlock();
735 auto bri = block->GetGraph()->GetBoundsRangeInfo();
736 auto range = bri->FindBoundsRange(block, inst);
737 bool canOverflow = true;
738 if constexpr (OPC == Opcode::NegOverflowAndZeroCheck) {
739 canOverflow = range.CanOverflowNeg(DataType::INT32);
740 } else {
741 canOverflow = range.CanOverflow(DataType::INT32);
742 }
743 if (!canOverflow) {
744 block->RemoveOverflowCheck(inst);
745 SetApplied();
746 return;
747 }
748 bool constInputs = true;
749 for (size_t i = 0; i < inst->GetInputsCount() - 1; ++i) {
750 constInputs &= inst->GetInput(i).GetInst()->IsConst();
751 }
752 if (constInputs) {
753 // replace by deopt
754 PushNewCheckMustThrow(inst);
755 return;
756 }
757 PushNewCheckForMoveOutOfLoop(inst);
758 }
759
FindSaveState(Loop * loop)760 Inst *ChecksElimination::FindSaveState(Loop *loop)
761 {
762 auto block = loop->GetPreHeader();
763 while (block != nullptr) {
764 for (const auto &inst : block->InstsSafeReverse()) {
765 if (inst->GetOpcode() == Opcode::SaveStateDeoptimize || inst->GetOpcode() == Opcode::SaveState) {
766 return inst;
767 }
768 }
769 auto next = block->GetDominator();
770 // The case when the dominant block is the head of a inner loop
771 if (next != nullptr && next->GetLoop()->GetOuterLoop() == block->GetLoop()) {
772 return nullptr;
773 }
774 block = next;
775 }
776 return nullptr;
777 }
778
FindOptimalSaveStateForHoist(Inst * inst,Inst ** optimalInsertAfter)779 Inst *ChecksElimination::FindOptimalSaveStateForHoist(Inst *inst, Inst **optimalInsertAfter)
780 {
781 ASSERT(inst->RequireState());
782 auto block = inst->GetBasicBlock();
783 if (block == nullptr) {
784 return nullptr;
785 }
786 auto loop = block->GetLoop();
787 *optimalInsertAfter = nullptr;
788
789 while (!loop->IsRoot() && !loop->GetHeader()->IsOsrEntry() && !loop->IsIrreducible()) {
790 for (auto backEdge : loop->GetBackEdges()) {
791 if (!block->IsDominate(backEdge)) {
792 // avoid taking checks out of slowpath
793 return *optimalInsertAfter;
794 }
795 }
796 // Find save state
797 Inst *ss = FindSaveState(loop);
798 if (ss == nullptr) {
799 return *optimalInsertAfter;
800 }
801 auto insertAfter = ss;
802
803 // Check that inputs are dominate on ss
804 bool inputsAreDominate = true;
805 for (size_t i = 0; i < inst->GetInputsCount() - 1; ++i) {
806 auto input = inst->GetInput(i).GetInst();
807 if (input->IsDominate(insertAfter)) {
808 continue;
809 }
810 if (insertAfter->GetBasicBlock() == input->GetBasicBlock()) {
811 insertAfter = input;
812 } else {
813 inputsAreDominate = false;
814 break;
815 }
816 }
817
818 if (!inputsAreDominate) {
819 return *optimalInsertAfter;
820 }
821 *optimalInsertAfter = insertAfter;
822 if (insertAfter != ss) {
823 // some inputs are dominate on insert_after but not dominate on ss, stop here
824 // the only case when return value is not equal to *optimal_insert_after
825 return ss;
826 }
827 block = loop->GetHeader(); // block will be used to check for hot path
828 loop = loop->GetOuterLoop();
829 }
830 return *optimalInsertAfter;
831 }
832
InsertInstAfter(Inst * inst,Inst * after,BasicBlock * block)833 void ChecksElimination::InsertInstAfter(Inst *inst, Inst *after, BasicBlock *block)
834 {
835 if (after->IsPhi()) {
836 block->PrependInst(inst);
837 } else {
838 block->InsertAfter(inst, after);
839 }
840 }
841
InsertBoundsCheckDeoptimization(ConditionCode cc,Inst * left,int64_t val,Inst * right,Inst * ss,Inst * insertAfter,Opcode newLeftOpcode)842 void ChecksElimination::InsertBoundsCheckDeoptimization(ConditionCode cc, Inst *left, int64_t val, Inst *right,
843 Inst *ss, Inst *insertAfter, Opcode newLeftOpcode)
844 {
845 auto block = insertAfter->GetBasicBlock();
846 Inst *newLeft = nullptr;
847 if (val == 0 && left != nullptr) {
848 newLeft = left;
849 } else if (left == nullptr) {
850 ASSERT(newLeftOpcode == Opcode::Add);
851 newLeft = GetGraph()->FindOrCreateConstant(val);
852 } else {
853 auto cnst = GetGraph()->FindOrCreateConstant(val);
854 newLeft = GetGraph()->CreateInst(newLeftOpcode);
855 newLeft->SetType(DataType::INT32);
856 newLeft->SetInput(0, left);
857 newLeft->SetInput(1, cnst);
858 if (newLeft->RequireState()) {
859 newLeft->SetSaveState(ss);
860 }
861 InsertInstAfter(newLeft, insertAfter, block);
862 insertAfter = newLeft;
863 }
864 auto deoptComp = GetGraph()->CreateInstCompare(DataType::BOOL, INVALID_PC, newLeft, right, DataType::INT32, cc);
865 auto deopt = GetGraph()->CreateInstDeoptimizeIf(ss->GetPc(), deoptComp, ss, DeoptimizeType::BOUNDS_CHECK);
866 InsertInstAfter(deoptComp, insertAfter, block);
867 block->InsertAfter(deopt, deoptComp);
868 }
869
InsertDeoptimization(ConditionCode cc,Inst * left,Inst * right,Inst * ss,Inst * insertAfter,DeoptimizeType deoptType)870 Inst *ChecksElimination::InsertDeoptimization(ConditionCode cc, Inst *left, Inst *right, Inst *ss, Inst *insertAfter,
871 DeoptimizeType deoptType)
872 {
873 auto deoptComp = GetGraph()->CreateInstCompare(DataType::BOOL, INVALID_PC, left, right, left->GetType(), cc);
874 auto deopt = GetGraph()->CreateInstDeoptimizeIf(ss->GetPc(), deoptComp, ss, deoptType);
875 auto block = insertAfter->GetBasicBlock();
876 block->InsertAfter(deoptComp, insertAfter);
877 block->InsertAfter(deopt, deoptComp);
878 return deopt;
879 }
880
FindLoopInfo(Loop * loop)881 std::optional<LoopInfo> ChecksElimination::FindLoopInfo(Loop *loop)
882 {
883 Inst *ss = FindSaveState(loop);
884 if (ss == nullptr) {
885 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "SaveState isn't founded";
886 return std::nullopt;
887 }
888 auto loopParser = CountableLoopParser(*loop);
889 if (auto loopInfo = loopParser.Parse()) {
890 auto loopInfoValue = loopInfo.value();
891 if (loopInfoValue.normalizedCc == CC_NE) {
892 return std::nullopt;
893 }
894 bool isHeadLoopExit = loopInfoValue.ifImm->GetBasicBlock() == loop->GetHeader();
895 bool hasPreHeaderCompare = CountableLoopParser::HasPreHeaderCompare(loop, loopInfoValue);
896 ASSERT(loopInfoValue.index->GetOpcode() == Opcode::Phi);
897 if (loopInfoValue.isInc) {
898 return std::make_tuple(loopInfoValue, ss, loopInfoValue.init, loopInfoValue.test,
899 loopInfoValue.normalizedCc == CC_LE ? CC_LE : CC_LT, isHeadLoopExit,
900 hasPreHeaderCompare);
901 }
902 return std::make_tuple(loopInfoValue, ss, loopInfoValue.test, loopInfoValue.init, CC_LE, isHeadLoopExit,
903 hasPreHeaderCompare);
904 }
905 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Not countable loop isn't supported";
906 return std::nullopt;
907 }
908
InsertNewLenArray(Inst * lenArray,Inst * ss)909 Inst *ChecksElimination::InsertNewLenArray(Inst *lenArray, Inst *ss)
910 {
911 if (lenArray->IsDominate(ss)) {
912 return lenArray;
913 }
914 if (lenArray->GetOpcode() == Opcode::LenArray) {
915 auto nullCheck = lenArray->GetInput(0).GetInst();
916 auto ref = lenArray->GetDataFlowInput(nullCheck);
917 if (ref->IsDominate(ss)) {
918 // Build nullcheck + lenarray before loop
919 auto nullcheck = GetGraph()->CreateInstNullCheck(DataType::REFERENCE, ss->GetPc(), ref, ss);
920 nullcheck->SetFlag(inst_flags::CAN_DEOPTIMIZE);
921 auto newLenArray = lenArray->Clone(GetGraph());
922 newLenArray->SetInput(0, nullcheck);
923 auto block = ss->GetBasicBlock();
924 block->InsertAfter(newLenArray, ss);
925 block->InsertAfter(nullcheck, ss);
926 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Builded new NullCheck(id=" << nullcheck->GetId()
927 << ") and LenArray(id=" << newLenArray->GetId() << ") before loop";
928 ChecksElimination::VisitNullCheck<true>(this, nullcheck);
929 return newLenArray;
930 }
931 }
932 return nullptr;
933 }
934
InsertDeoptimizationForIndexOverflow(CountableLoopInfo * countableLoopInfo,BoundsRange indexUpperRange,Inst * ss)935 void ChecksElimination::InsertDeoptimizationForIndexOverflow(CountableLoopInfo *countableLoopInfo,
936 BoundsRange indexUpperRange, Inst *ss)
937 {
938 auto loopCc = countableLoopInfo->normalizedCc;
939 if (loopCc == CC_LT || loopCc == CC_LE) {
940 auto loopUpper = countableLoopInfo->test;
941 ASSERT(countableLoopInfo->constStep < INT64_MAX);
942 auto step = static_cast<int64_t>(countableLoopInfo->constStep);
943 auto indexType = countableLoopInfo->index->GetType();
944 ASSERT(indexType == DataType::INT32);
945 auto maxUpper = BoundsRange::GetMax(indexType) - step + (loopCc == CC_LT ? 1 : 0);
946 auto bri = loopUpper->GetBasicBlock()->GetGraph()->GetBoundsRangeInfo();
947 auto loopUpperRange = bri->FindBoundsRange(countableLoopInfo->index->GetBasicBlock(), loopUpper);
948 // Upper bound of loop index assuming (index + maxAdd < lenArray)
949 indexUpperRange = indexUpperRange.Add(BoundsRange(step)).FitInType(indexType);
950 if (!BoundsRange(maxUpper).IsMoreOrEqual(loopUpperRange) && indexUpperRange.IsMaxRange(indexType)) {
951 // loop index can overflow
952 Inst *insertAfter = loopUpper->IsDominate(ss) ? ss : loopUpper;
953 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Build deoptimize for loop index overflow";
954 // Create deoptimize if loop index can become negative
955 InsertBoundsCheckDeoptimization(ConditionCode::CC_LT, nullptr, maxUpper, loopUpper, ss, insertAfter);
956 }
957 }
958 }
959
NeedUpperDeoptimization(BasicBlock * header,Inst * lenArray,BoundsRange lenArrayRange,Inst * upper,BoundsRange upperRange,int64_t maxAdd,ConditionCode cc,bool * insertNewLenArray)960 bool ChecksElimination::NeedUpperDeoptimization(BasicBlock *header, Inst *lenArray, BoundsRange lenArrayRange,
961 Inst *upper, BoundsRange upperRange, int64_t maxAdd, ConditionCode cc,
962 bool *insertNewLenArray)
963 {
964 if (maxAdd == std::numeric_limits<int64_t>::min()) {
965 return false;
966 }
967 if (upperRange.Add(BoundsRange(maxAdd)).IsLess(lenArrayRange)) {
968 return false;
969 }
970 auto bri = GetGraph()->GetBoundsRangeInfo();
971 auto newUpper = upper;
972 auto newMaxAdd = maxAdd;
973 int64_t upperAdd = 0;
974 if (IsInstIncOrDec(upper)) {
975 upperAdd = GetInc(upper);
976 newMaxAdd += upperAdd;
977 newUpper = upper->GetInput(0).GetInst();
978 }
979 if (lenArray == newUpper) {
980 if (newMaxAdd < 0 || (newMaxAdd == 0 && cc == CC_LT)) {
981 return false;
982 }
983 }
984 auto useUpperLen = upperAdd >= 0 || !upperRange.IsMaxRange(upper->GetType());
985 if (useUpperLen && newMaxAdd <= 0) {
986 auto newUpperRange = bri->FindBoundsRange(header, newUpper);
987 if (newUpperRange.GetLenArray() == lenArray) {
988 return false;
989 }
990 }
991 *insertNewLenArray = newUpper != lenArray;
992 return true;
993 }
994
TryInsertDeoptimizationForLargeStep(ConditionCode cc,Inst * resultLenArray,Inst * lower,Inst * upper,int64_t maxAdd,Inst * insertDeoptAfter,Inst * ss,uint64_t constStep)995 bool ChecksElimination::TryInsertDeoptimizationForLargeStep(ConditionCode cc, Inst *resultLenArray, Inst *lower,
996 Inst *upper, int64_t maxAdd, Inst *insertDeoptAfter,
997 Inst *ss, uint64_t constStep)
998 {
999 auto block = insertDeoptAfter->GetBasicBlock();
1000 if (!lower->IsDominate(insertDeoptAfter)) {
1001 if (lower->GetBasicBlock() == block) {
1002 insertDeoptAfter = lower;
1003 } else {
1004 return false;
1005 }
1006 }
1007 auto subValue = lower;
1008 if (cc == CC_LT) {
1009 subValue = GetGraph()->CreateInstAdd(DataType::INT32, INVALID_PC, lower, GetGraph()->FindOrCreateConstant(1));
1010 InsertInstAfter(subValue, insertDeoptAfter, block);
1011 insertDeoptAfter = subValue;
1012 }
1013 auto sub = GetGraph()->CreateInstSub(DataType::INT32, INVALID_PC, upper, subValue);
1014 InsertInstAfter(sub, insertDeoptAfter, block);
1015 auto mod = GetGraph()->CreateInstMod(DataType::INT32, INVALID_PC, sub, GetGraph()->FindOrCreateConstant(constStep));
1016 block->InsertAfter(mod, sub);
1017 if (resultLenArray == upper) {
1018 auto maxAddConst = GetGraph()->FindOrCreateConstant(maxAdd);
1019 // (upper - lower [- 1]) % step </<= maxAdd
1020 InsertBoundsCheckDeoptimization(cc, mod, 0, maxAddConst, ss, mod, Opcode::NOP);
1021 } else {
1022 // result_len_array - maxAdd </<= upper - (upper - lower [- 1]) % step
1023 auto maxIndexValue = GetGraph()->CreateInstSub(DataType::INT32, INVALID_PC, upper, mod);
1024 block->InsertAfter(maxIndexValue, mod);
1025 auto opcode = maxAdd > 0 ? Opcode::Sub : Opcode::SubOverflowCheck;
1026 InsertBoundsCheckDeoptimization(cc, resultLenArray, maxAdd, maxIndexValue, ss, maxIndexValue, opcode);
1027 }
1028 return true;
1029 }
1030
TryInsertDeoptimization(LoopInfo loopInfo,Inst * lenArray,int64_t maxAdd,int64_t minAdd,bool hasCheckInHeader)1031 bool ChecksElimination::TryInsertDeoptimization(LoopInfo loopInfo, Inst *lenArray, int64_t maxAdd, int64_t minAdd,
1032 bool hasCheckInHeader)
1033 {
1034 auto [countable_loop_info, ss, lower, upper, cc, is_head_loop_exit, has_pre_header_compare] = loopInfo;
1035 ASSERT(cc == CC_LT || cc == CC_LE);
1036 auto bri = GetGraph()->GetBoundsRangeInfo();
1037 auto header = countable_loop_info.index->GetBasicBlock();
1038 auto upperRange = bri->FindBoundsRange(header, upper);
1039 auto lowerRange = bri->FindBoundsRange(header, lower);
1040 auto lenArrayRange = bri->FindBoundsRange(header, lenArray);
1041 auto hasCheckBeforeExit = hasCheckInHeader || !is_head_loop_exit;
1042 if (!has_pre_header_compare && !lowerRange.IsLess(upperRange) && hasCheckBeforeExit) {
1043 // if lower > upper, removing BoundsCheck may be wrong for the first iteration
1044 return false;
1045 }
1046 uint64_t lowerInc = (countable_loop_info.normalizedCc == CC_GT ? 1 : 0);
1047 bool needLowerDeopt = (minAdd != std::numeric_limits<int64_t>::max()) &&
1048 !lowerRange.Add(BoundsRange(minAdd)).Add(BoundsRange(lowerInc)).IsNotNegative();
1049 bool insertLowerDeopt = lower->IsDominate(ss);
1050 if (needLowerDeopt && !insertLowerDeopt) {
1051 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Unable to build deoptimize for lower value";
1052 return false;
1053 }
1054
1055 bool insertNewLenArray;
1056 if (NeedUpperDeoptimization(header, lenArray, lenArrayRange, upper, upperRange, maxAdd, cc, &insertNewLenArray)) {
1057 auto resultLenArray = insertNewLenArray ? InsertNewLenArray(lenArray, ss) : lenArray;
1058 if (resultLenArray == nullptr) {
1059 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Unable to build deoptimize for upper value";
1060 return false;
1061 }
1062 auto insertDeoptAfter = lenArray != resultLenArray ? resultLenArray : ss;
1063 if (!upper->IsDominate(insertDeoptAfter)) {
1064 insertDeoptAfter = upper;
1065 }
1066 ASSERT(insertDeoptAfter->GetBasicBlock()->IsDominate(header));
1067 if (insertDeoptAfter->GetBasicBlock() == header) {
1068 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Unable to build deoptimize for upper value";
1069 return false;
1070 }
1071 auto constStep = countable_loop_info.constStep;
1072 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Try to build deoptimize for upper value";
1073 if (constStep == 1 ||
1074 (countable_loop_info.normalizedCc == CC_GT || countable_loop_info.normalizedCc == CC_GE)) {
1075 auto opcode = maxAdd > 0 ? Opcode::Sub : Opcode::SubOverflowCheck;
1076 // Create deoptimize if result_len_array - maxAdd <=(<) upper
1077 // result_len_array is >= 0, so if maxAdd > 0, overflow is not possible
1078 // that's why we do not add maxAdd to upper instead
1079 InsertBoundsCheckDeoptimization(cc, resultLenArray, maxAdd, upper, ss, insertDeoptAfter, opcode);
1080 } else if (lowerRange.IsConst() && lowerRange.GetLeft() == 0 && countable_loop_info.normalizedCc == CC_LT &&
1081 resultLenArray == upper && maxAdd == static_cast<int64_t>(constStep) - 1) {
1082 // For (int i = 0; i < len; i += x) process(a[i], ..., a[i + x - 1])
1083 // deoptimize if len % x != 0
1084 auto zeroConst = GetGraph()->FindOrCreateConstant(0);
1085 InsertBoundsCheckDeoptimization(ConditionCode::CC_NE, resultLenArray, constStep, zeroConst, ss,
1086 insertDeoptAfter, Opcode::Mod);
1087 } else if (!TryInsertDeoptimizationForLargeStep(cc, resultLenArray, lower, upper, maxAdd, insertDeoptAfter, ss,
1088 constStep)) {
1089 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Unable to build deoptimize for upper value with step > 1";
1090 return false;
1091 }
1092 }
1093 InsertDeoptimizationForIndexOverflow(&countable_loop_info, lenArrayRange.Sub(BoundsRange(maxAdd)), ss);
1094 if (needLowerDeopt) {
1095 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Build deoptimize for lower value";
1096 // Create deoptimize if lower < 0 (or -1 for loop with CC_GT)
1097 auto lowerConst = GetGraph()->FindOrCreateConstant(-lowerInc);
1098 InsertBoundsCheckDeoptimization(ConditionCode::CC_LT, lower, minAdd, lowerConst, ss, ss);
1099 }
1100 return true;
1101 }
1102
HoistLoopInvariantBoundsChecks(Inst * lenArray,GroupedBoundsChecks * indexBoundschecks,Loop * loop)1103 void ChecksElimination::HoistLoopInvariantBoundsChecks(Inst *lenArray, GroupedBoundsChecks *indexBoundschecks,
1104 Loop *loop)
1105 {
1106 auto lenArrLoop = lenArray->GetBasicBlock()->GetLoop();
1107 // lenArray isn't loop invariant
1108 if (lenArrLoop == loop) {
1109 return;
1110 }
1111 for (auto &[index, boundChecks, max, min] : *indexBoundschecks) {
1112 // Check that index is loop invariant, if index is nullptr it means that it was a constant
1113 if (index != nullptr && index->GetBasicBlock()->GetLoop() == loop) {
1114 continue;
1115 }
1116 for (auto boundsCheck : boundChecks) {
1117 PushNewCheckForMoveOutOfLoop(boundsCheck);
1118 }
1119 }
1120 }
1121
ProcessingGroupBoundsCheck(GroupedBoundsChecks * indexBoundschecks,LoopInfo loopInfo,Inst * lenArray)1122 void ChecksElimination::ProcessingGroupBoundsCheck(GroupedBoundsChecks *indexBoundschecks, LoopInfo loopInfo,
1123 Inst *lenArray)
1124 {
1125 auto phiIndex = std::get<0>(loopInfo).index;
1126 auto phiIndexIt = std::find_if(indexBoundschecks->begin(), indexBoundschecks->end(),
1127 [=](auto p) { return std::get<0>(p) == phiIndex; });
1128 if (phiIndexIt == indexBoundschecks->end()) {
1129 HoistLoopInvariantBoundsChecks(lenArray, indexBoundschecks, phiIndex->GetBasicBlock()->GetLoop());
1130 return;
1131 }
1132 const auto &instsToDelete = std::get<1U>(*phiIndexIt);
1133 auto maxAdd = std::get<2U>(*phiIndexIt);
1134 auto minAdd = std::get<3U>(*phiIndexIt);
1135 ASSERT(!instsToDelete.empty());
1136 bool hasCheckInHeader = false;
1137 for (const auto &inst : instsToDelete) {
1138 if (inst->GetBasicBlock() == phiIndex->GetBasicBlock()) {
1139 hasCheckInHeader = true;
1140 }
1141 }
1142 if (TryInsertDeoptimization(loopInfo, lenArray, maxAdd, minAdd, hasCheckInHeader)) {
1143 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Delete group of BoundsChecks";
1144 // Delete bounds checks instructions
1145 for (const auto &inst : instsToDelete) {
1146 ReplaceUsersAndRemoveCheck(inst, inst->GetInput(1).GetInst());
1147 auto it = std::find_if(indexBoundschecks->begin(), indexBoundschecks->end(),
1148 [=](auto p) { return std::get<0>(p) == inst->GetInput(1).GetInst(); });
1149 if (it != indexBoundschecks->end()) {
1150 std::get<0>(*it) = nullptr;
1151 std::get<1>(*it).clear();
1152 }
1153 }
1154 }
1155 }
1156
ProcessingLoop(Loop * loop,LoopNotFullyRedundantBoundsCheck * lenarrIndexChecks)1157 void ChecksElimination::ProcessingLoop(Loop *loop, LoopNotFullyRedundantBoundsCheck *lenarrIndexChecks)
1158 {
1159 auto loopInfo = FindLoopInfo(loop);
1160 if (loopInfo == std::nullopt) {
1161 return;
1162 }
1163 for (auto it = lenarrIndexChecks->rbegin(); it != lenarrIndexChecks->rend(); it++) {
1164 ProcessingGroupBoundsCheck(&it->second, loopInfo.value(), it->first);
1165 }
1166 }
1167
ReplaceBoundsCheckToDeoptimizationBeforeLoop()1168 void ChecksElimination::ReplaceBoundsCheckToDeoptimizationBeforeLoop()
1169 {
1170 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Start ReplaceBoundsCheckToDeoptimizationBeforeLoop";
1171 for (auto &[loop, lenarrIndexChecks] : boundsChecks_) {
1172 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Processing loop with id = " << loop->GetId();
1173 if (loop->IsRoot()) {
1174 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Skip root loop";
1175 continue;
1176 }
1177 ProcessingLoop(loop, &lenarrIndexChecks);
1178 }
1179 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Finish ReplaceBoundsCheckToDeoptimizationBeforeLoop";
1180 }
1181
MoveCheckOutOfLoop()1182 void ChecksElimination::MoveCheckOutOfLoop()
1183 {
1184 for (auto inst : checksForMoveOutOfLoop_) {
1185 Inst *insertAfter = nullptr;
1186 auto ss = FindOptimalSaveStateForHoist(inst, &insertAfter);
1187 if (ss == nullptr) {
1188 continue;
1189 }
1190 if (inst->GetOpcode() == Opcode::CheckCast) {
1191 // input object can become null after hoisting
1192 inst->CastToCheckCast()->SetOmitNullCheck(false);
1193 }
1194 ASSERT(insertAfter != nullptr);
1195 ASSERT(ss->GetBasicBlock() == insertAfter->GetBasicBlock());
1196 auto block = inst->GetBasicBlock();
1197 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Move check " << GetOpcodeString(inst->GetOpcode())
1198 << " with id = " << inst->GetId() << " from bb " << block->GetId() << " to bb "
1199 << ss->GetBasicBlock()->GetId();
1200 block->EraseInst(inst);
1201 ss->GetBasicBlock()->InsertAfter(inst, insertAfter);
1202 inst->SetSaveState(ss);
1203 inst->SetPc(ss->GetPc());
1204 inst->SetFlag(inst_flags::CAN_DEOPTIMIZE);
1205 SetApplied();
1206 }
1207 }
1208
FindSaveState(const InstVector & instsToDelete)1209 Inst *ChecksElimination::FindSaveState(const InstVector &instsToDelete)
1210 {
1211 for (auto boundsCheck : instsToDelete) {
1212 bool isDominate = true;
1213 for (auto boundsCheckTest : instsToDelete) {
1214 if (boundsCheck == boundsCheckTest) {
1215 continue;
1216 }
1217 isDominate &= boundsCheck->IsDominate(boundsCheckTest);
1218 }
1219 if (isDominate) {
1220 constexpr auto IMM_2 = 2;
1221 return boundsCheck->GetInput(IMM_2).GetInst();
1222 }
1223 }
1224 return nullptr;
1225 }
1226
ReplaceOneBoundsCheckToDeoptimizationInLoop(std::pair<Loop *,LoopNotFullyRedundantBoundsCheck> & item)1227 void ChecksElimination::ReplaceOneBoundsCheckToDeoptimizationInLoop(
1228 std::pair<Loop *, LoopNotFullyRedundantBoundsCheck> &item)
1229 {
1230 for (auto &[lenArray, indexBoundchecks] : item.second) {
1231 for (auto &[index, instsToDelete, max, min] : indexBoundchecks) {
1232 constexpr auto MIN_BOUNDSCHECKS_NUM = 2;
1233 if (instsToDelete.size() <= MIN_BOUNDSCHECKS_NUM) {
1234 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Skip small group of BoundsChecks";
1235 continue;
1236 }
1237 // Try to replace more than 2 bounds checks to deoptimization in loop
1238 auto saveState = FindSaveState(instsToDelete);
1239 if (saveState == nullptr) {
1240 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "SaveState isn't founded";
1241 continue;
1242 }
1243 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Replace group of BoundsChecks on deoptimization in loop";
1244 auto insertAfter = lenArray->IsDominate(saveState) ? saveState : lenArray;
1245 if (max != std::numeric_limits<int64_t>::min()) {
1246 // Create deoptimize if max_index >= lenArray
1247 InsertBoundsCheckDeoptimization(ConditionCode::CC_GE, index, max, lenArray, saveState, insertAfter);
1248 }
1249 if (index != nullptr && min != std::numeric_limits<int64_t>::max()) {
1250 // Create deoptimize if min_index < 0
1251 auto zeroConst = GetGraph()->FindOrCreateConstant(0);
1252 InsertBoundsCheckDeoptimization(ConditionCode::CC_LT, index, min, zeroConst, saveState, insertAfter);
1253 } else {
1254 // No lower check needed based on BoundsAnalysis
1255 // if index is null, group of bounds checks consists of constants
1256 ASSERT(min >= 0);
1257 }
1258 for (const auto &inst : instsToDelete) {
1259 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Delete group of BoundsChecks";
1260 ReplaceUsersAndRemoveCheck(inst, inst->GetInput(1).GetInst());
1261 }
1262 }
1263 }
1264 }
1265
ReplaceBoundsCheckToDeoptimizationInLoop()1266 void ChecksElimination::ReplaceBoundsCheckToDeoptimizationInLoop()
1267 {
1268 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Start ReplaceBoundsCheckToDeoptimizationInLoop";
1269 for (auto &item : boundsChecks_) {
1270 ReplaceOneBoundsCheckToDeoptimizationInLoop(item);
1271 }
1272 COMPILER_LOG(DEBUG, CHECKS_ELIM) << "Finish ReplaceBoundsCheckToDeoptimizationInLoop";
1273 }
1274
ReplaceCheckMustThrowByUnconditionalDeoptimize()1275 void ChecksElimination::ReplaceCheckMustThrowByUnconditionalDeoptimize()
1276 {
1277 for (auto &inst : checksMustThrow_) {
1278 auto block = inst->GetBasicBlock();
1279 if (block != nullptr) {
1280 COMPILER_LOG(DEBUG, CHECKS_ELIM)
1281 << "Replace check with id = " << inst->GetId() << " by uncondition deoptimize";
1282 block->ReplaceInstByDeoptimize(inst);
1283 SetApplied();
1284 SetLoopDeleted();
1285 }
1286 }
1287 }
1288
1289 } // namespace ark::compiler
1290