1 /*
2 * Copyright (C) 2015 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "reference_type_propagation.h"
18
19 #include "art_field-inl.h"
20 #include "art_method-inl.h"
21 #include "base/arena_allocator.h"
22 #include "base/pointer_size.h"
23 #include "base/scoped_arena_allocator.h"
24 #include "base/scoped_arena_containers.h"
25 #include "class_linker-inl.h"
26 #include "class_root-inl.h"
27 #include "handle_cache-inl.h"
28 #include "handle_scope-inl.h"
29 #include "mirror/class-inl.h"
30 #include "mirror/dex_cache.h"
31 #include "scoped_thread_state_change-inl.h"
32
33 namespace art HIDDEN {
34
FindDexCacheWithHint(Thread * self,const DexFile & dex_file,Handle<mirror::DexCache> hint_dex_cache)35 static inline ObjPtr<mirror::DexCache> FindDexCacheWithHint(
36 Thread* self, const DexFile& dex_file, Handle<mirror::DexCache> hint_dex_cache)
37 REQUIRES_SHARED(Locks::mutator_lock_) {
38 if (LIKELY(hint_dex_cache->GetDexFile() == &dex_file)) {
39 return hint_dex_cache.Get();
40 } else {
41 return Runtime::Current()->GetClassLinker()->FindDexCache(self, dex_file);
42 }
43 }
44
45 class ReferenceTypePropagation::RTPVisitor final : public HGraphDelegateVisitor {
46 public:
RTPVisitor(HGraph * graph,Handle<mirror::DexCache> hint_dex_cache,bool is_first_run)47 RTPVisitor(HGraph* graph, Handle<mirror::DexCache> hint_dex_cache, bool is_first_run)
48 : HGraphDelegateVisitor(graph),
49 hint_dex_cache_(hint_dex_cache),
50 allocator_(graph->GetArenaStack()),
51 worklist_(allocator_.Adapter(kArenaAllocReferenceTypePropagation)),
52 is_first_run_(is_first_run) {
53 worklist_.reserve(kDefaultWorklistSize);
54 }
55
56 void VisitDeoptimize(HDeoptimize* deopt) override;
57 void VisitNewInstance(HNewInstance* new_instance) override;
58 void VisitLoadClass(HLoadClass* load_class) override;
59 void VisitInstanceOf(HInstanceOf* load_class) override;
60 void VisitClinitCheck(HClinitCheck* clinit_check) override;
61 void VisitLoadMethodHandle(HLoadMethodHandle* instr) override;
62 void VisitLoadMethodType(HLoadMethodType* instr) override;
63 void VisitLoadString(HLoadString* instr) override;
64 void VisitLoadException(HLoadException* instr) override;
65 void VisitNewArray(HNewArray* instr) override;
66 void VisitParameterValue(HParameterValue* instr) override;
67 void VisitInstanceFieldGet(HInstanceFieldGet* instr) override;
68 void VisitStaticFieldGet(HStaticFieldGet* instr) override;
69 void VisitUnresolvedInstanceFieldGet(HUnresolvedInstanceFieldGet* instr) override;
70 void VisitUnresolvedStaticFieldGet(HUnresolvedStaticFieldGet* instr) override;
71 void VisitInvoke(HInvoke* instr) override;
72 void VisitArrayGet(HArrayGet* instr) override;
73 void VisitCheckCast(HCheckCast* instr) override;
74 void VisitBoundType(HBoundType* instr) override;
75 void VisitNullCheck(HNullCheck* instr) override;
76 void VisitPhi(HPhi* phi) override;
77
78 void VisitBasicBlock(HBasicBlock* block) override;
79 void ProcessWorklist();
80
81 private:
82 void UpdateFieldAccessTypeInfo(HInstruction* instr, const FieldInfo& info);
83 void SetClassAsTypeInfo(HInstruction* instr, ObjPtr<mirror::Class> klass, bool is_exact)
84 REQUIRES_SHARED(Locks::mutator_lock_);
85 void BoundTypeForIfNotNull(HBasicBlock* block);
86 static void BoundTypeForIfInstanceOf(HBasicBlock* block);
87 static bool UpdateNullability(HInstruction* instr);
88 static void UpdateBoundType(HBoundType* bound_type) REQUIRES_SHARED(Locks::mutator_lock_);
89 void UpdateArrayGet(HArrayGet* instr) REQUIRES_SHARED(Locks::mutator_lock_);
90 void UpdatePhi(HPhi* phi) REQUIRES_SHARED(Locks::mutator_lock_);
91 bool UpdateReferenceTypeInfo(HInstruction* instr);
92 void UpdateReferenceTypeInfo(HInstruction* instr,
93 dex::TypeIndex type_idx,
94 const DexFile& dex_file,
95 bool is_exact);
96
97 // Returns true if this is an instruction we might need to recursively update.
98 // The types are (live) Phi, BoundType, ArrayGet, and NullCheck
99 static constexpr bool IsUpdateable(const HInstruction* instr);
100 void AddToWorklist(HInstruction* instruction);
101 void AddDependentInstructionsToWorklist(HInstruction* instruction);
102
GetHandleCache()103 HandleCache* GetHandleCache() {
104 return GetGraph()->GetHandleCache();
105 }
106
107 static constexpr size_t kDefaultWorklistSize = 8;
108
109 Handle<mirror::DexCache> hint_dex_cache_;
110
111 // Use local allocator for allocating memory.
112 ScopedArenaAllocator allocator_;
113 ScopedArenaVector<HInstruction*> worklist_;
114 const bool is_first_run_;
115
116 friend class ReferenceTypePropagation;
117 };
118
ReferenceTypePropagation(HGraph * graph,Handle<mirror::DexCache> hint_dex_cache,bool is_first_run,const char * name)119 ReferenceTypePropagation::ReferenceTypePropagation(HGraph* graph,
120 Handle<mirror::DexCache> hint_dex_cache,
121 bool is_first_run,
122 const char* name)
123 : HOptimization(graph, name), hint_dex_cache_(hint_dex_cache), is_first_run_(is_first_run) {}
124
Visit(HInstruction * instruction)125 void ReferenceTypePropagation::Visit(HInstruction* instruction) {
126 RTPVisitor visitor(graph_, hint_dex_cache_, is_first_run_);
127 instruction->Accept(&visitor);
128 }
129
Visit(ArrayRef<HInstruction * const> instructions)130 void ReferenceTypePropagation::Visit(ArrayRef<HInstruction* const> instructions) {
131 RTPVisitor visitor(graph_, hint_dex_cache_, is_first_run_);
132 for (HInstruction* instruction : instructions) {
133 if (instruction->IsPhi()) {
134 // Need to force phis to recalculate null-ness.
135 instruction->AsPhi()->SetCanBeNull(false);
136 }
137 }
138 for (HInstruction* instruction : instructions) {
139 instruction->Accept(&visitor);
140 // We don't know if the instruction list is ordered in the same way normal
141 // visiting would be so we need to process every instruction manually.
142 if (RTPVisitor::IsUpdateable(instruction)) {
143 visitor.AddToWorklist(instruction);
144 }
145 }
146 visitor.ProcessWorklist();
147 }
148
149 // Check if we should create a bound type for the given object at the specified
150 // position. Because of inlining and the fact we run RTP more than once and we
151 // might have a HBoundType already. If we do, we should not create a new one.
152 // In this case we also assert that there are no other uses of the object (except
153 // the bound type) dominated by the specified dominator_instr or dominator_block.
ShouldCreateBoundType(HInstruction * position,HInstruction * obj,ReferenceTypeInfo upper_bound,HInstruction * dominator_instr,HBasicBlock * dominator_block)154 static bool ShouldCreateBoundType(HInstruction* position,
155 HInstruction* obj,
156 ReferenceTypeInfo upper_bound,
157 HInstruction* dominator_instr,
158 HBasicBlock* dominator_block)
159 REQUIRES_SHARED(Locks::mutator_lock_) {
160 // If the position where we should insert the bound type is not already a
161 // a bound type then we need to create one.
162 if (position == nullptr || !position->IsBoundType()) {
163 return true;
164 }
165
166 HBoundType* existing_bound_type = position->AsBoundType();
167 if (existing_bound_type->GetUpperBound().IsSupertypeOf(upper_bound)) {
168 if (kIsDebugBuild) {
169 // Check that the existing HBoundType dominates all the uses.
170 for (const HUseListNode<HInstruction*>& use : obj->GetUses()) {
171 HInstruction* user = use.GetUser();
172 if (dominator_instr != nullptr) {
173 DCHECK(!dominator_instr->StrictlyDominates(user)
174 || user == existing_bound_type
175 || existing_bound_type->StrictlyDominates(user));
176 } else if (dominator_block != nullptr) {
177 DCHECK(!dominator_block->Dominates(user->GetBlock())
178 || user == existing_bound_type
179 || existing_bound_type->StrictlyDominates(user));
180 }
181 }
182 }
183 } else {
184 // TODO: if the current bound type is a refinement we could update the
185 // existing_bound_type with the a new upper limit. However, we also need to
186 // update its users and have access to the work list.
187 }
188 return false;
189 }
190
191 // Helper method to bound the type of `receiver` for all instructions dominated
192 // by `start_block`, or `start_instruction` if `start_block` is null. The new
193 // bound type will have its upper bound be `class_rti`.
BoundTypeIn(HInstruction * receiver,HBasicBlock * start_block,HInstruction * start_instruction,const ReferenceTypeInfo & class_rti)194 static void BoundTypeIn(HInstruction* receiver,
195 HBasicBlock* start_block,
196 HInstruction* start_instruction,
197 const ReferenceTypeInfo& class_rti) {
198 // We only need to bound the type if we have uses in the relevant block.
199 // So start with null and create the HBoundType lazily, only if it's needed.
200 HBoundType* bound_type = nullptr;
201 DCHECK(!receiver->IsLoadClass()) << "We should not replace HLoadClass instructions";
202 const HUseList<HInstruction*>& uses = receiver->GetUses();
203 for (auto it = uses.begin(), end = uses.end(); it != end; /* ++it below */) {
204 HInstruction* user = it->GetUser();
205 size_t index = it->GetIndex();
206 // Increment `it` now because `*it` may disappear thanks to user->ReplaceInput().
207 ++it;
208 bool dominates = (start_instruction != nullptr)
209 ? start_instruction->StrictlyDominates(user)
210 : start_block->Dominates(user->GetBlock());
211 if (!dominates) {
212 continue;
213 }
214 if (bound_type == nullptr) {
215 ScopedObjectAccess soa(Thread::Current());
216 HInstruction* insert_point = (start_instruction != nullptr)
217 ? start_instruction->GetNext()
218 : start_block->GetFirstInstruction();
219 if (ShouldCreateBoundType(
220 insert_point, receiver, class_rti, start_instruction, start_block)) {
221 bound_type = new (start_block->GetGraph()->GetAllocator()) HBoundType(receiver);
222 bound_type->SetUpperBound(class_rti, /* can_be_null= */ false);
223 start_block->InsertInstructionBefore(bound_type, insert_point);
224 // To comply with the RTP algorithm, don't type the bound type just yet, it will
225 // be handled in RTPVisitor::VisitBoundType.
226 } else {
227 // We already have a bound type on the position we would need to insert
228 // the new one. The existing bound type should dominate all the users
229 // (dchecked) so there's no need to continue.
230 break;
231 }
232 }
233 user->ReplaceInput(bound_type, index);
234 }
235 // If the receiver is a null check, also bound the type of the actual
236 // receiver.
237 if (receiver->IsNullCheck()) {
238 BoundTypeIn(receiver->InputAt(0), start_block, start_instruction, class_rti);
239 }
240 }
241
242 // Recognize the patterns:
243 // if (obj.shadow$_klass_ == Foo.class) ...
244 // deoptimize if (obj.shadow$_klass_ == Foo.class)
BoundTypeForClassCheck(HInstruction * check)245 static void BoundTypeForClassCheck(HInstruction* check) {
246 if (!check->IsIf() && !check->IsDeoptimize()) {
247 return;
248 }
249 HInstruction* compare = check->InputAt(0);
250 if (!compare->IsEqual() && !compare->IsNotEqual()) {
251 return;
252 }
253 HInstruction* input_one = compare->InputAt(0);
254 HInstruction* input_two = compare->InputAt(1);
255 HLoadClass* load_class = input_one->IsLoadClass()
256 ? input_one->AsLoadClass()
257 : input_two->AsLoadClassOrNull();
258 if (load_class == nullptr) {
259 return;
260 }
261
262 ReferenceTypeInfo class_rti = load_class->GetLoadedClassRTI();
263 if (!class_rti.IsValid()) {
264 // We have loaded an unresolved class. Don't bother bounding the type.
265 return;
266 }
267
268 HInstruction* field_get = (load_class == input_one) ? input_two : input_one;
269 if (!field_get->IsInstanceFieldGet()) {
270 return;
271 }
272 HInstruction* receiver = field_get->InputAt(0);
273 ReferenceTypeInfo receiver_type = receiver->GetReferenceTypeInfo();
274 if (receiver_type.IsExact()) {
275 // If we already know the receiver type, don't bother updating its users.
276 return;
277 }
278
279 if (field_get->AsInstanceFieldGet()->GetFieldInfo().GetField() !=
280 WellKnownClasses::java_lang_Object_shadowKlass) {
281 return;
282 }
283
284 if (check->IsIf()) {
285 HBasicBlock* trueBlock = compare->IsEqual()
286 ? check->AsIf()->IfTrueSuccessor()
287 : check->AsIf()->IfFalseSuccessor();
288 BoundTypeIn(receiver, trueBlock, /* start_instruction= */ nullptr, class_rti);
289 } else {
290 DCHECK(check->IsDeoptimize());
291 if (compare->IsEqual() && check->AsDeoptimize()->GuardsAnInput()) {
292 check->SetReferenceTypeInfo(class_rti);
293 }
294 }
295 }
296
Run()297 bool ReferenceTypePropagation::Run() {
298 DCHECK(Thread::Current() != nullptr)
299 << "ReferenceTypePropagation requires the use of Thread::Current(). Make sure you have a "
300 << "Runtime initialized before calling this optimization pass";
301 RTPVisitor visitor(graph_, hint_dex_cache_, is_first_run_);
302
303 // To properly propagate type info we need to visit in the dominator-based order.
304 // Reverse post order guarantees a node's dominators are visited first.
305 // We take advantage of this order in `VisitBasicBlock`.
306 for (HBasicBlock* block : graph_->GetReversePostOrder()) {
307 visitor.VisitBasicBlock(block);
308 }
309
310 visitor.ProcessWorklist();
311 return true;
312 }
313
VisitBasicBlock(HBasicBlock * block)314 void ReferenceTypePropagation::RTPVisitor::VisitBasicBlock(HBasicBlock* block) {
315 // Handle Phis first as there might be instructions in the same block who depend on them.
316 VisitPhis(block);
317
318 // Handle instructions. Since RTP may add HBoundType instructions just after the
319 // last visited instruction, use `HInstructionIteratorHandleChanges` iterator.
320 VisitNonPhiInstructionsHandleChanges(block);
321
322 // Add extra nodes to bound types.
323 BoundTypeForIfNotNull(block);
324 BoundTypeForIfInstanceOf(block);
325 BoundTypeForClassCheck(block->GetLastInstruction());
326 }
327
BoundTypeForIfNotNull(HBasicBlock * block)328 void ReferenceTypePropagation::RTPVisitor::BoundTypeForIfNotNull(HBasicBlock* block) {
329 HIf* ifInstruction = block->GetLastInstruction()->AsIfOrNull();
330 if (ifInstruction == nullptr) {
331 return;
332 }
333 HInstruction* ifInput = ifInstruction->InputAt(0);
334 if (!ifInput->IsNotEqual() && !ifInput->IsEqual()) {
335 return;
336 }
337 HInstruction* input0 = ifInput->InputAt(0);
338 HInstruction* input1 = ifInput->InputAt(1);
339 HInstruction* obj = nullptr;
340
341 if (input1->IsNullConstant()) {
342 obj = input0;
343 } else if (input0->IsNullConstant()) {
344 obj = input1;
345 } else {
346 return;
347 }
348
349 if (!obj->CanBeNull() || obj->IsNullConstant()) {
350 // Null check is dead code and will be removed by DCE.
351 return;
352 }
353 DCHECK(!obj->IsLoadClass()) << "We should not replace HLoadClass instructions";
354
355 // We only need to bound the type if we have uses in the relevant block.
356 // So start with null and create the HBoundType lazily, only if it's needed.
357 HBasicBlock* notNullBlock = ifInput->IsNotEqual()
358 ? ifInstruction->IfTrueSuccessor()
359 : ifInstruction->IfFalseSuccessor();
360
361 ReferenceTypeInfo object_rti =
362 ReferenceTypeInfo::Create(GetHandleCache()->GetObjectClassHandle(), /* is_exact= */ false);
363
364 BoundTypeIn(obj, notNullBlock, /* start_instruction= */ nullptr, object_rti);
365 }
366
367 // Returns true if one of the patterns below has been recognized. If so, the
368 // InstanceOf instruction together with the true branch of `ifInstruction` will
369 // be returned using the out parameters.
370 // Recognized patterns:
371 // (1) patterns equivalent to `if (obj instanceof X)`
372 // (a) InstanceOf -> Equal to 1 -> If
373 // (b) InstanceOf -> NotEqual to 0 -> If
374 // (c) InstanceOf -> If
375 // (2) patterns equivalent to `if (!(obj instanceof X))`
376 // (a) InstanceOf -> Equal to 0 -> If
377 // (b) InstanceOf -> NotEqual to 1 -> If
378 // (c) InstanceOf -> BooleanNot -> If
MatchIfInstanceOf(HIf * ifInstruction,HInstanceOf ** instanceOf,HBasicBlock ** trueBranch)379 static bool MatchIfInstanceOf(HIf* ifInstruction,
380 /* out */ HInstanceOf** instanceOf,
381 /* out */ HBasicBlock** trueBranch) {
382 HInstruction* input = ifInstruction->InputAt(0);
383
384 if (input->IsEqual()) {
385 HInstruction* rhs = input->AsEqual()->GetConstantRight();
386 if (rhs != nullptr) {
387 HInstruction* lhs = input->AsEqual()->GetLeastConstantLeft();
388 if (lhs->IsInstanceOf() && rhs->IsIntConstant()) {
389 if (rhs->AsIntConstant()->IsTrue()) {
390 // Case (1a)
391 *trueBranch = ifInstruction->IfTrueSuccessor();
392 } else if (rhs->AsIntConstant()->IsFalse()) {
393 // Case (2a)
394 *trueBranch = ifInstruction->IfFalseSuccessor();
395 } else {
396 // Sometimes we see a comparison of instance-of with a constant which is neither 0 nor 1.
397 // In those cases, we cannot do the match if+instance-of.
398 return false;
399 }
400 *instanceOf = lhs->AsInstanceOf();
401 return true;
402 }
403 }
404 } else if (input->IsNotEqual()) {
405 HInstruction* rhs = input->AsNotEqual()->GetConstantRight();
406 if (rhs != nullptr) {
407 HInstruction* lhs = input->AsNotEqual()->GetLeastConstantLeft();
408 if (lhs->IsInstanceOf() && rhs->IsIntConstant()) {
409 if (rhs->AsIntConstant()->IsFalse()) {
410 // Case (1b)
411 *trueBranch = ifInstruction->IfTrueSuccessor();
412 } else if (rhs->AsIntConstant()->IsTrue()) {
413 // Case (2b)
414 *trueBranch = ifInstruction->IfFalseSuccessor();
415 } else {
416 // Sometimes we see a comparison of instance-of with a constant which is neither 0 nor 1.
417 // In those cases, we cannot do the match if+instance-of.
418 return false;
419 }
420 *instanceOf = lhs->AsInstanceOf();
421 return true;
422 }
423 }
424 } else if (input->IsInstanceOf()) {
425 // Case (1c)
426 *instanceOf = input->AsInstanceOf();
427 *trueBranch = ifInstruction->IfTrueSuccessor();
428 return true;
429 } else if (input->IsBooleanNot()) {
430 HInstruction* not_input = input->InputAt(0);
431 if (not_input->IsInstanceOf()) {
432 // Case (2c)
433 *instanceOf = not_input->AsInstanceOf();
434 *trueBranch = ifInstruction->IfFalseSuccessor();
435 return true;
436 }
437 }
438
439 return false;
440 }
441
442 // Detects if `block` is the True block for the pattern
443 // `if (x instanceof ClassX) { }`
444 // If that's the case insert an HBoundType instruction to bound the type of `x`
445 // to `ClassX` in the scope of the dominated blocks.
BoundTypeForIfInstanceOf(HBasicBlock * block)446 void ReferenceTypePropagation::RTPVisitor::BoundTypeForIfInstanceOf(HBasicBlock* block) {
447 HIf* ifInstruction = block->GetLastInstruction()->AsIfOrNull();
448 if (ifInstruction == nullptr) {
449 return;
450 }
451
452 // Try to recognize common `if (instanceof)` and `if (!instanceof)` patterns.
453 HInstanceOf* instanceOf = nullptr;
454 HBasicBlock* instanceOfTrueBlock = nullptr;
455 if (!MatchIfInstanceOf(ifInstruction, &instanceOf, &instanceOfTrueBlock)) {
456 return;
457 }
458
459 ReferenceTypeInfo class_rti = instanceOf->GetTargetClassRTI();
460 if (!class_rti.IsValid()) {
461 // We have loaded an unresolved class. Don't bother bounding the type.
462 return;
463 }
464
465 HInstruction* obj = instanceOf->InputAt(0);
466 if (obj->GetReferenceTypeInfo().IsExact() && !obj->IsPhi()) {
467 // This method is being called while doing a fixed-point calculation
468 // over phis. Non-phis instruction whose type is already known do
469 // not need to be bound to another type.
470 // Not that this also prevents replacing `HLoadClass` with a `HBoundType`.
471 // `HCheckCast` and `HInstanceOf` expect a `HLoadClass` as a second
472 // input.
473 return;
474 }
475
476 {
477 ScopedObjectAccess soa(Thread::Current());
478 if (!class_rti.GetTypeHandle()->CannotBeAssignedFromOtherTypes()) {
479 class_rti = ReferenceTypeInfo::Create(class_rti.GetTypeHandle(), /* is_exact= */ false);
480 }
481 }
482 BoundTypeIn(obj, instanceOfTrueBlock, /* start_instruction= */ nullptr, class_rti);
483 }
484
SetClassAsTypeInfo(HInstruction * instr,ObjPtr<mirror::Class> klass,bool is_exact)485 void ReferenceTypePropagation::RTPVisitor::SetClassAsTypeInfo(HInstruction* instr,
486 ObjPtr<mirror::Class> klass,
487 bool is_exact) {
488 if (instr->IsInvokeStaticOrDirect() && instr->AsInvokeStaticOrDirect()->IsStringInit()) {
489 // Calls to String.<init> are replaced with a StringFactory.
490 if (kIsDebugBuild) {
491 HInvokeStaticOrDirect* invoke = instr->AsInvokeStaticOrDirect();
492 ClassLinker* cl = Runtime::Current()->GetClassLinker();
493 Thread* self = Thread::Current();
494 StackHandleScope<2> hs(self);
495 const DexFile& dex_file = *invoke->GetResolvedMethodReference().dex_file;
496 uint32_t dex_method_index = invoke->GetResolvedMethodReference().index;
497 Handle<mirror::DexCache> dex_cache(
498 hs.NewHandle(FindDexCacheWithHint(self, dex_file, hint_dex_cache_)));
499 // Use a null loader, the target method is in a boot classpath dex file.
500 Handle<mirror::ClassLoader> loader(hs.NewHandle<mirror::ClassLoader>(nullptr));
501 ArtMethod* method = cl->ResolveMethodId(dex_method_index, dex_cache, loader);
502 DCHECK(method != nullptr);
503 ObjPtr<mirror::Class> declaring_class = method->GetDeclaringClass();
504 DCHECK(declaring_class != nullptr);
505 DCHECK(declaring_class->IsStringClass())
506 << "Expected String class: " << declaring_class->PrettyDescriptor();
507 DCHECK(method->IsConstructor())
508 << "Expected String.<init>: " << method->PrettyMethod();
509 }
510 instr->SetReferenceTypeInfo(
511 ReferenceTypeInfo::Create(GetHandleCache()->GetStringClassHandle(), /* is_exact= */ true));
512 } else if (IsAdmissible(klass)) {
513 ReferenceTypeInfo::TypeHandle handle = GetHandleCache()->NewHandle(klass);
514 is_exact = is_exact || handle->CannotBeAssignedFromOtherTypes();
515 instr->SetReferenceTypeInfo(ReferenceTypeInfo::Create(handle, is_exact));
516 } else {
517 instr->SetReferenceTypeInfo(GetGraph()->GetInexactObjectRti());
518 }
519 }
520
VisitDeoptimize(HDeoptimize * instr)521 void ReferenceTypePropagation::RTPVisitor::VisitDeoptimize(HDeoptimize* instr) {
522 BoundTypeForClassCheck(instr);
523 }
524
UpdateReferenceTypeInfo(HInstruction * instr,dex::TypeIndex type_idx,const DexFile & dex_file,bool is_exact)525 void ReferenceTypePropagation::RTPVisitor::UpdateReferenceTypeInfo(HInstruction* instr,
526 dex::TypeIndex type_idx,
527 const DexFile& dex_file,
528 bool is_exact) {
529 DCHECK_EQ(instr->GetType(), DataType::Type::kReference);
530
531 ScopedObjectAccess soa(Thread::Current());
532 StackHandleScope<2> hs(soa.Self());
533 Handle<mirror::DexCache> dex_cache =
534 hs.NewHandle(FindDexCacheWithHint(soa.Self(), dex_file, hint_dex_cache_));
535 Handle<mirror::ClassLoader> loader = hs.NewHandle(dex_cache->GetClassLoader());
536 ObjPtr<mirror::Class> klass = Runtime::Current()->GetClassLinker()->ResolveType(
537 type_idx, dex_cache, loader);
538 DCHECK_EQ(klass == nullptr, soa.Self()->IsExceptionPending());
539 soa.Self()->ClearException(); // Clean up the exception left by type resolution if any.
540 SetClassAsTypeInfo(instr, klass, is_exact);
541 }
542
VisitNewInstance(HNewInstance * instr)543 void ReferenceTypePropagation::RTPVisitor::VisitNewInstance(HNewInstance* instr) {
544 ScopedObjectAccess soa(Thread::Current());
545 SetClassAsTypeInfo(instr, instr->GetLoadClass()->GetClass().Get(), /* is_exact= */ true);
546 }
547
VisitNewArray(HNewArray * instr)548 void ReferenceTypePropagation::RTPVisitor::VisitNewArray(HNewArray* instr) {
549 ScopedObjectAccess soa(Thread::Current());
550 SetClassAsTypeInfo(instr, instr->GetLoadClass()->GetClass().Get(), /* is_exact= */ true);
551 }
552
VisitParameterValue(HParameterValue * instr)553 void ReferenceTypePropagation::RTPVisitor::VisitParameterValue(HParameterValue* instr) {
554 // We check if the existing type is valid: the inliner may have set it.
555 if (instr->GetType() == DataType::Type::kReference && !instr->GetReferenceTypeInfo().IsValid()) {
556 UpdateReferenceTypeInfo(instr,
557 instr->GetTypeIndex(),
558 instr->GetDexFile(),
559 /* is_exact= */ false);
560 }
561 }
562
UpdateFieldAccessTypeInfo(HInstruction * instr,const FieldInfo & info)563 void ReferenceTypePropagation::RTPVisitor::UpdateFieldAccessTypeInfo(HInstruction* instr,
564 const FieldInfo& info) {
565 if (instr->GetType() != DataType::Type::kReference) {
566 return;
567 }
568
569 ScopedObjectAccess soa(Thread::Current());
570 ObjPtr<mirror::Class> klass;
571
572 // The field is unknown only during tests.
573 if (info.GetField() != nullptr) {
574 klass = info.GetField()->LookupResolvedType();
575 }
576
577 SetClassAsTypeInfo(instr, klass, /* is_exact= */ false);
578 }
579
VisitInstanceFieldGet(HInstanceFieldGet * instr)580 void ReferenceTypePropagation::RTPVisitor::VisitInstanceFieldGet(HInstanceFieldGet* instr) {
581 UpdateFieldAccessTypeInfo(instr, instr->GetFieldInfo());
582 }
583
VisitStaticFieldGet(HStaticFieldGet * instr)584 void ReferenceTypePropagation::RTPVisitor::VisitStaticFieldGet(HStaticFieldGet* instr) {
585 UpdateFieldAccessTypeInfo(instr, instr->GetFieldInfo());
586 }
587
VisitUnresolvedInstanceFieldGet(HUnresolvedInstanceFieldGet * instr)588 void ReferenceTypePropagation::RTPVisitor::VisitUnresolvedInstanceFieldGet(
589 HUnresolvedInstanceFieldGet* instr) {
590 // TODO: Use descriptor to get the actual type.
591 if (instr->GetFieldType() == DataType::Type::kReference) {
592 instr->SetReferenceTypeInfo(GetGraph()->GetInexactObjectRti());
593 }
594 }
595
VisitUnresolvedStaticFieldGet(HUnresolvedStaticFieldGet * instr)596 void ReferenceTypePropagation::RTPVisitor::VisitUnresolvedStaticFieldGet(
597 HUnresolvedStaticFieldGet* instr) {
598 // TODO: Use descriptor to get the actual type.
599 if (instr->GetFieldType() == DataType::Type::kReference) {
600 instr->SetReferenceTypeInfo(GetGraph()->GetInexactObjectRti());
601 }
602 }
603
VisitLoadClass(HLoadClass * instr)604 void ReferenceTypePropagation::RTPVisitor::VisitLoadClass(HLoadClass* instr) {
605 ScopedObjectAccess soa(Thread::Current());
606 if (IsAdmissible(instr->GetClass().Get())) {
607 instr->SetValidLoadedClassRTI();
608 }
609 instr->SetReferenceTypeInfo(
610 ReferenceTypeInfo::Create(GetHandleCache()->GetClassClassHandle(), /* is_exact= */ true));
611 }
612
VisitInstanceOf(HInstanceOf * instr)613 void ReferenceTypePropagation::RTPVisitor::VisitInstanceOf(HInstanceOf* instr) {
614 ScopedObjectAccess soa(Thread::Current());
615 if (IsAdmissible(instr->GetClass().Get())) {
616 instr->SetValidTargetClassRTI();
617 }
618 }
619
VisitClinitCheck(HClinitCheck * instr)620 void ReferenceTypePropagation::RTPVisitor::VisitClinitCheck(HClinitCheck* instr) {
621 instr->SetReferenceTypeInfo(instr->InputAt(0)->GetReferenceTypeInfo());
622 }
623
VisitLoadMethodHandle(HLoadMethodHandle * instr)624 void ReferenceTypePropagation::RTPVisitor::VisitLoadMethodHandle(HLoadMethodHandle* instr) {
625 instr->SetReferenceTypeInfo(ReferenceTypeInfo::Create(
626 GetHandleCache()->GetMethodHandleClassHandle(), /* is_exact= */ true));
627 }
628
VisitLoadMethodType(HLoadMethodType * instr)629 void ReferenceTypePropagation::RTPVisitor::VisitLoadMethodType(HLoadMethodType* instr) {
630 instr->SetReferenceTypeInfo(ReferenceTypeInfo::Create(
631 GetHandleCache()->GetMethodTypeClassHandle(), /* is_exact= */ true));
632 }
633
VisitLoadString(HLoadString * instr)634 void ReferenceTypePropagation::RTPVisitor::VisitLoadString(HLoadString* instr) {
635 instr->SetReferenceTypeInfo(
636 ReferenceTypeInfo::Create(GetHandleCache()->GetStringClassHandle(), /* is_exact= */ true));
637 }
638
VisitLoadException(HLoadException * instr)639 void ReferenceTypePropagation::RTPVisitor::VisitLoadException(HLoadException* instr) {
640 DCHECK(instr->GetBlock()->IsCatchBlock());
641 TryCatchInformation* catch_info = instr->GetBlock()->GetTryCatchInformation();
642
643 if (catch_info->IsValidTypeIndex()) {
644 UpdateReferenceTypeInfo(instr,
645 catch_info->GetCatchTypeIndex(),
646 catch_info->GetCatchDexFile(),
647 /* is_exact= */ false);
648 } else {
649 instr->SetReferenceTypeInfo(ReferenceTypeInfo::Create(
650 GetHandleCache()->GetThrowableClassHandle(), /* is_exact= */ false));
651 }
652 }
653
VisitNullCheck(HNullCheck * instr)654 void ReferenceTypePropagation::RTPVisitor::VisitNullCheck(HNullCheck* instr) {
655 ReferenceTypeInfo parent_rti = instr->InputAt(0)->GetReferenceTypeInfo();
656 if (parent_rti.IsValid()) {
657 instr->SetReferenceTypeInfo(parent_rti);
658 }
659 }
660
VisitBoundType(HBoundType * instr)661 void ReferenceTypePropagation::RTPVisitor::VisitBoundType(HBoundType* instr) {
662 ReferenceTypeInfo class_rti = instr->GetUpperBound();
663 if (class_rti.IsValid()) {
664 ScopedObjectAccess soa(Thread::Current());
665 // Narrow the type as much as possible.
666 HInstruction* obj = instr->InputAt(0);
667 ReferenceTypeInfo obj_rti = obj->GetReferenceTypeInfo();
668 if (class_rti.IsExact()) {
669 instr->SetReferenceTypeInfo(class_rti);
670 } else if (obj_rti.IsValid()) {
671 if (class_rti.IsSupertypeOf(obj_rti)) {
672 // Object type is more specific.
673 instr->SetReferenceTypeInfo(obj_rti);
674 } else {
675 // Upper bound is more specific, or unrelated to the object's type.
676 // Note that the object might then be exact, and we know the code dominated by this
677 // bound type is dead. To not confuse potential other optimizations, we mark
678 // the bound as non-exact.
679 instr->SetReferenceTypeInfo(
680 ReferenceTypeInfo::Create(class_rti.GetTypeHandle(), /* is_exact= */ false));
681 }
682 } else {
683 // Object not typed yet. Leave BoundType untyped for now rather than
684 // assign the type conservatively.
685 }
686 instr->SetCanBeNull(obj->CanBeNull() && instr->GetUpperCanBeNull());
687 } else {
688 // The owner of the BoundType was already visited. If the class is unresolved,
689 // the BoundType should have been removed from the data flow and this method
690 // should remove it from the graph.
691 DCHECK(!instr->HasUses());
692 instr->GetBlock()->RemoveInstruction(instr);
693 }
694 }
695
VisitCheckCast(HCheckCast * check_cast)696 void ReferenceTypePropagation::RTPVisitor::VisitCheckCast(HCheckCast* check_cast) {
697 HBoundType* bound_type = check_cast->GetNext()->AsBoundTypeOrNull();
698 if (bound_type == nullptr || bound_type->GetUpperBound().IsValid()) {
699 // The next instruction is not an uninitialized BoundType. This must be
700 // an RTP pass after SsaBuilder and we do not need to do anything.
701 return;
702 }
703 DCHECK_EQ(bound_type->InputAt(0), check_cast->InputAt(0));
704
705 ScopedObjectAccess soa(Thread::Current());
706 Handle<mirror::Class> klass = check_cast->GetClass();
707 if (IsAdmissible(klass.Get())) {
708 DCHECK(is_first_run_);
709 check_cast->SetValidTargetClassRTI();
710 // This is the first run of RTP and class is resolved.
711 bool is_exact = klass->CannotBeAssignedFromOtherTypes();
712 bound_type->SetUpperBound(ReferenceTypeInfo::Create(klass, is_exact),
713 /* CheckCast succeeds for nulls. */ true);
714 } else {
715 // This is the first run of RTP and class is unresolved. Remove the binding.
716 // The instruction itself is removed in VisitBoundType so as to not
717 // invalidate HInstructionIterator.
718 bound_type->ReplaceWith(bound_type->InputAt(0));
719 }
720 }
721
VisitPhi(HPhi * phi)722 void ReferenceTypePropagation::RTPVisitor::VisitPhi(HPhi* phi) {
723 if (phi->IsDead() || phi->GetType() != DataType::Type::kReference) {
724 return;
725 }
726
727 if (phi->GetBlock()->IsLoopHeader()) {
728 // Set the initial type for the phi. Use the non back edge input for reaching
729 // a fixed point faster.
730 HInstruction* first_input = phi->InputAt(0);
731 ReferenceTypeInfo first_input_rti = first_input->GetReferenceTypeInfo();
732 if (first_input_rti.IsValid() && !first_input->IsNullConstant()) {
733 phi->SetCanBeNull(first_input->CanBeNull());
734 phi->SetReferenceTypeInfo(first_input_rti);
735 }
736 AddToWorklist(phi);
737 } else {
738 // Eagerly compute the type of the phi, for quicker convergence. Note
739 // that we don't need to add users to the worklist because we are
740 // doing a reverse post-order visit, therefore either the phi users are
741 // non-loop phi and will be visited later in the visit, or are loop-phis,
742 // and they are already in the work list.
743 UpdateNullability(phi);
744 UpdateReferenceTypeInfo(phi);
745 }
746 }
747
FixUpSelectType(HSelect * select,HandleCache * handle_cache)748 void ReferenceTypePropagation::FixUpSelectType(HSelect* select, HandleCache* handle_cache) {
749 ReferenceTypeInfo false_rti = select->GetFalseValue()->GetReferenceTypeInfo();
750 ReferenceTypeInfo true_rti = select->GetTrueValue()->GetReferenceTypeInfo();
751 ReferenceTypeInfo rti = ReferenceTypeInfo::CreateInvalid();
752 ScopedObjectAccess soa(Thread::Current());
753 select->SetReferenceTypeInfo(MergeTypes(false_rti, true_rti, handle_cache));
754 }
755
MergeTypes(const ReferenceTypeInfo & a,const ReferenceTypeInfo & b,HandleCache * handle_cache)756 ReferenceTypeInfo ReferenceTypePropagation::MergeTypes(const ReferenceTypeInfo& a,
757 const ReferenceTypeInfo& b,
758 HandleCache* handle_cache) {
759 if (!b.IsValid()) {
760 return a;
761 }
762 if (!a.IsValid()) {
763 return b;
764 }
765
766 bool is_exact = a.IsExact() && b.IsExact();
767 ReferenceTypeInfo::TypeHandle result_type_handle;
768 ReferenceTypeInfo::TypeHandle a_type_handle = a.GetTypeHandle();
769 ReferenceTypeInfo::TypeHandle b_type_handle = b.GetTypeHandle();
770 bool a_is_interface = a_type_handle->IsInterface();
771 bool b_is_interface = b_type_handle->IsInterface();
772
773 if (a.GetTypeHandle().Get() == b.GetTypeHandle().Get()) {
774 result_type_handle = a_type_handle;
775 } else if (a.IsSupertypeOf(b)) {
776 result_type_handle = a_type_handle;
777 is_exact = false;
778 } else if (b.IsSupertypeOf(a)) {
779 result_type_handle = b_type_handle;
780 is_exact = false;
781 } else if (!a_is_interface && !b_is_interface) {
782 result_type_handle =
783 handle_cache->NewHandle(a_type_handle->GetCommonSuperClass(b_type_handle));
784 is_exact = false;
785 } else {
786 // This can happen if:
787 // - both types are interfaces. TODO(calin): implement
788 // - one is an interface, the other a class, and the type does not implement the interface
789 // e.g:
790 // void foo(Interface i, boolean cond) {
791 // Object o = cond ? i : new Object();
792 // }
793 result_type_handle = handle_cache->GetObjectClassHandle();
794 is_exact = false;
795 }
796
797 return ReferenceTypeInfo::Create(result_type_handle, is_exact);
798 }
799
UpdateArrayGet(HArrayGet * instr)800 void ReferenceTypePropagation::RTPVisitor::UpdateArrayGet(HArrayGet* instr) {
801 DCHECK_EQ(DataType::Type::kReference, instr->GetType());
802
803 ReferenceTypeInfo parent_rti = instr->InputAt(0)->GetReferenceTypeInfo();
804 if (!parent_rti.IsValid()) {
805 return;
806 }
807
808 Handle<mirror::Class> handle = parent_rti.GetTypeHandle();
809 if (handle->IsObjectArrayClass() && IsAdmissible(handle->GetComponentType())) {
810 ReferenceTypeInfo::TypeHandle component_handle =
811 GetHandleCache()->NewHandle(handle->GetComponentType());
812 bool is_exact = component_handle->CannotBeAssignedFromOtherTypes();
813 instr->SetReferenceTypeInfo(ReferenceTypeInfo::Create(component_handle, is_exact));
814 } else {
815 // We don't know what the parent actually is, so we fallback to object.
816 instr->SetReferenceTypeInfo(GetGraph()->GetInexactObjectRti());
817 }
818 }
819
UpdateReferenceTypeInfo(HInstruction * instr)820 bool ReferenceTypePropagation::RTPVisitor::UpdateReferenceTypeInfo(HInstruction* instr) {
821 ScopedObjectAccess soa(Thread::Current());
822
823 ReferenceTypeInfo previous_rti = instr->GetReferenceTypeInfo();
824 if (instr->IsBoundType()) {
825 UpdateBoundType(instr->AsBoundType());
826 } else if (instr->IsPhi()) {
827 UpdatePhi(instr->AsPhi());
828 } else if (instr->IsNullCheck()) {
829 ReferenceTypeInfo parent_rti = instr->InputAt(0)->GetReferenceTypeInfo();
830 if (parent_rti.IsValid()) {
831 instr->SetReferenceTypeInfo(parent_rti);
832 }
833 } else if (instr->IsArrayGet()) {
834 // TODO: consider if it's worth "looking back" and binding the input object
835 // to an array type.
836 UpdateArrayGet(instr->AsArrayGet());
837 } else {
838 LOG(FATAL) << "Invalid instruction (should not get here)";
839 }
840
841 return !previous_rti.IsEqual(instr->GetReferenceTypeInfo());
842 }
843
VisitInvoke(HInvoke * instr)844 void ReferenceTypePropagation::RTPVisitor::VisitInvoke(HInvoke* instr) {
845 if (instr->GetType() != DataType::Type::kReference) {
846 return;
847 }
848
849 ScopedObjectAccess soa(Thread::Current());
850 // FIXME: Treat InvokePolymorphic separately, as we can get a more specific return type from
851 // protoId than the one obtained from the resolved method.
852 ArtMethod* method = instr->GetResolvedMethod();
853 ObjPtr<mirror::Class> klass = (method == nullptr) ? nullptr : method->LookupResolvedReturnType();
854 SetClassAsTypeInfo(instr, klass, /* is_exact= */ false);
855 }
856
VisitArrayGet(HArrayGet * instr)857 void ReferenceTypePropagation::RTPVisitor::VisitArrayGet(HArrayGet* instr) {
858 if (instr->GetType() != DataType::Type::kReference) {
859 return;
860 }
861
862 ScopedObjectAccess soa(Thread::Current());
863 UpdateArrayGet(instr);
864 if (!instr->GetReferenceTypeInfo().IsValid()) {
865 worklist_.push_back(instr);
866 }
867 }
868
UpdateBoundType(HBoundType * instr)869 void ReferenceTypePropagation::RTPVisitor::UpdateBoundType(HBoundType* instr) {
870 ReferenceTypeInfo input_rti = instr->InputAt(0)->GetReferenceTypeInfo();
871 if (!input_rti.IsValid()) {
872 return; // No new info yet.
873 }
874
875 ReferenceTypeInfo upper_bound_rti = instr->GetUpperBound();
876 if (upper_bound_rti.IsExact()) {
877 instr->SetReferenceTypeInfo(upper_bound_rti);
878 } else if (upper_bound_rti.IsSupertypeOf(input_rti)) {
879 // input is more specific.
880 instr->SetReferenceTypeInfo(input_rti);
881 } else {
882 // upper_bound is more specific or unrelated.
883 // Note that the object might then be exact, and we know the code dominated by this
884 // bound type is dead. To not confuse potential other optimizations, we mark
885 // the bound as non-exact.
886 instr->SetReferenceTypeInfo(
887 ReferenceTypeInfo::Create(upper_bound_rti.GetTypeHandle(), /* is_exact= */ false));
888 }
889 }
890
891 // NullConstant inputs are ignored during merging as they do not provide any useful information.
892 // If all the inputs are NullConstants then the type of the phi will be set to Object.
UpdatePhi(HPhi * instr)893 void ReferenceTypePropagation::RTPVisitor::UpdatePhi(HPhi* instr) {
894 DCHECK(instr->IsLive());
895
896 HInputsRef inputs = instr->GetInputs();
897 size_t first_input_index_not_null = 0;
898 while (first_input_index_not_null < inputs.size() &&
899 inputs[first_input_index_not_null]->IsNullConstant()) {
900 first_input_index_not_null++;
901 }
902 if (first_input_index_not_null == inputs.size()) {
903 // All inputs are NullConstants, set the type to object.
904 // This may happen in the presence of inlining.
905 instr->SetReferenceTypeInfo(instr->GetBlock()->GetGraph()->GetInexactObjectRti());
906 return;
907 }
908
909 ReferenceTypeInfo new_rti = instr->InputAt(first_input_index_not_null)->GetReferenceTypeInfo();
910
911 if (new_rti.IsValid() && new_rti.IsObjectClass() && !new_rti.IsExact()) {
912 // Early return if we are Object and inexact.
913 instr->SetReferenceTypeInfo(new_rti);
914 return;
915 }
916
917 for (size_t i = first_input_index_not_null + 1; i < inputs.size(); i++) {
918 if (inputs[i]->IsNullConstant()) {
919 continue;
920 }
921 new_rti = MergeTypes(new_rti, inputs[i]->GetReferenceTypeInfo(), GetHandleCache());
922 if (new_rti.IsValid() && new_rti.IsObjectClass()) {
923 if (!new_rti.IsExact()) {
924 break;
925 } else {
926 continue;
927 }
928 }
929 }
930
931 if (new_rti.IsValid()) {
932 instr->SetReferenceTypeInfo(new_rti);
933 }
934 }
935
IsUpdateable(const HInstruction * instr)936 constexpr bool ReferenceTypePropagation::RTPVisitor::IsUpdateable(const HInstruction* instr) {
937 return (instr->IsPhi() && instr->AsPhi()->IsLive()) ||
938 instr->IsBoundType() ||
939 instr->IsNullCheck() ||
940 instr->IsArrayGet();
941 }
942
943 // Re-computes and updates the nullability of the instruction. Returns whether or
944 // not the nullability was changed.
UpdateNullability(HInstruction * instr)945 bool ReferenceTypePropagation::RTPVisitor::UpdateNullability(HInstruction* instr) {
946 DCHECK(IsUpdateable(instr));
947
948 if (!instr->IsPhi() && !instr->IsBoundType()) {
949 return false;
950 }
951
952 bool existing_can_be_null = instr->CanBeNull();
953 if (instr->IsPhi()) {
954 HPhi* phi = instr->AsPhi();
955 bool new_can_be_null = false;
956 for (HInstruction* input : phi->GetInputs()) {
957 if (input->CanBeNull()) {
958 new_can_be_null = true;
959 break;
960 }
961 }
962 phi->SetCanBeNull(new_can_be_null);
963 } else if (instr->IsBoundType()) {
964 HBoundType* bound_type = instr->AsBoundType();
965 bound_type->SetCanBeNull(instr->InputAt(0)->CanBeNull() && bound_type->GetUpperCanBeNull());
966 }
967 return existing_can_be_null != instr->CanBeNull();
968 }
969
ProcessWorklist()970 void ReferenceTypePropagation::RTPVisitor::ProcessWorklist() {
971 while (!worklist_.empty()) {
972 HInstruction* instruction = worklist_.back();
973 worklist_.pop_back();
974 bool updated_nullability = UpdateNullability(instruction);
975 bool updated_reference_type = UpdateReferenceTypeInfo(instruction);
976 if (updated_nullability || updated_reference_type) {
977 AddDependentInstructionsToWorklist(instruction);
978 }
979 }
980 }
981
AddToWorklist(HInstruction * instruction)982 void ReferenceTypePropagation::RTPVisitor::AddToWorklist(HInstruction* instruction) {
983 DCHECK_EQ(instruction->GetType(), DataType::Type::kReference)
984 << instruction->DebugName() << ":" << instruction->GetType();
985 worklist_.push_back(instruction);
986 }
987
AddDependentInstructionsToWorklist(HInstruction * instruction)988 void ReferenceTypePropagation::RTPVisitor::AddDependentInstructionsToWorklist(
989 HInstruction* instruction) {
990 for (const HUseListNode<HInstruction*>& use : instruction->GetUses()) {
991 HInstruction* user = use.GetUser();
992 if ((user->IsPhi() && user->AsPhi()->IsLive())
993 || user->IsBoundType()
994 || user->IsNullCheck()
995 || (user->IsArrayGet() && (user->GetType() == DataType::Type::kReference))) {
996 AddToWorklist(user);
997 }
998 }
999 }
1000
1001 } // namespace art
1002