1 /*
2 * Copyright (C) 2013 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #ifndef ART_RUNTIME_GC_HEAP_INL_H_
18 #define ART_RUNTIME_GC_HEAP_INL_H_
19
20 #include "heap.h"
21
22 #include "allocation_listener.h"
23 #include "base/quasi_atomic.h"
24 #include "base/time_utils.h"
25 #include "gc/accounting/atomic_stack.h"
26 #include "gc/accounting/card_table-inl.h"
27 #include "gc/allocation_record.h"
28 #include "gc/collector/semi_space.h"
29 #include "gc/space/bump_pointer_space-inl.h"
30 #include "gc/space/dlmalloc_space-inl.h"
31 #include "gc/space/large_object_space.h"
32 #include "gc/space/region_space-inl.h"
33 #include "gc/space/rosalloc_space-inl.h"
34 #include "handle_scope-inl.h"
35 #include "obj_ptr-inl.h"
36 #include "runtime.h"
37 #include "thread-inl.h"
38 #include "verify_object.h"
39 #include "write_barrier-inl.h"
40
41 namespace art {
42 namespace gc {
43
44 template <bool kInstrumented, bool kCheckLargeObject, typename PreFenceVisitor>
AllocObjectWithAllocator(Thread * self,ObjPtr<mirror::Class> klass,size_t byte_count,AllocatorType allocator,const PreFenceVisitor & pre_fence_visitor)45 inline mirror::Object* Heap::AllocObjectWithAllocator(Thread* self,
46 ObjPtr<mirror::Class> klass,
47 size_t byte_count,
48 AllocatorType allocator,
49 const PreFenceVisitor& pre_fence_visitor) {
50 auto no_suspend_pre_fence_visitor =
51 [&pre_fence_visitor](auto... x) REQUIRES_SHARED(Locks::mutator_lock_) {
52 ScopedAssertNoThreadSuspension sants("No thread suspension during pre-fence visitor");
53 pre_fence_visitor(x...);
54 };
55
56 if (kIsDebugBuild) {
57 CheckPreconditionsForAllocObject(klass, byte_count);
58 // Since allocation can cause a GC which will need to SuspendAll, make sure all allocations are
59 // done in the runnable state where suspension is expected.
60 CHECK_EQ(self->GetState(), ThreadState::kRunnable);
61 self->AssertThreadSuspensionIsAllowable();
62 self->AssertNoPendingException();
63 // Make sure to preserve klass.
64 StackHandleScope<1> hs(self);
65 HandleWrapperObjPtr<mirror::Class> h = hs.NewHandleWrapper(&klass);
66 self->PoisonObjectPointers();
67 }
68 auto pre_object_allocated = [&]() REQUIRES_SHARED(Locks::mutator_lock_)
69 REQUIRES(!Roles::uninterruptible_ /* only suspends if kInstrumented */) {
70 if constexpr (kInstrumented) {
71 AllocationListener* l = alloc_listener_.load(std::memory_order_seq_cst);
72 if (UNLIKELY(l != nullptr) && UNLIKELY(l->HasPreAlloc())) {
73 StackHandleScope<1> hs(self);
74 HandleWrapperObjPtr<mirror::Class> h_klass(hs.NewHandleWrapper(&klass));
75 l->PreObjectAllocated(self, h_klass, &byte_count);
76 }
77 }
78 };
79 ObjPtr<mirror::Object> obj;
80 // bytes allocated for the (individual) object.
81 size_t bytes_allocated;
82 size_t usable_size;
83 size_t new_num_bytes_allocated = 0;
84 bool need_gc = false;
85 uint32_t starting_gc_num; // o.w. GC number at which we observed need for GC.
86 {
87 // Bytes allocated that includes bulk thread-local buffer allocations in addition to direct
88 // non-TLAB object allocations. Only set for non-thread-local allocation,
89 size_t bytes_tl_bulk_allocated = 0u;
90 // Do the initial pre-alloc
91 // TODO: Consider what happens if the allocator is switched while suspended here.
92 pre_object_allocated();
93
94 // Need to check that we aren't the large object allocator since the large object allocation
95 // code path includes this function. If we didn't check we would have an infinite loop.
96 if (kCheckLargeObject && UNLIKELY(ShouldAllocLargeObject(klass, byte_count))) {
97 // AllocLargeObject can suspend and will recall PreObjectAllocated if needed.
98 obj = AllocLargeObject<kInstrumented, PreFenceVisitor>(self, &klass, byte_count,
99 pre_fence_visitor);
100 if (obj != nullptr) {
101 return obj.Ptr();
102 }
103 // There should be an OOM exception, since we are retrying, clear it.
104 self->ClearException();
105
106 // If the large object allocation failed, try to use the normal spaces (main space,
107 // non moving space). This can happen if there is significant virtual address space
108 // fragmentation.
109 // kInstrumented may be out of date, so recurse without large object checking, rather than
110 // continue.
111 return AllocObjectWithAllocator</*kInstrumented=*/ true, /*kCheckLargeObject=*/ false>
112 (self, klass, byte_count, GetUpdatedAllocator(allocator), pre_fence_visitor);
113 }
114 ScopedAssertNoThreadSuspension ants("Called PreObjectAllocated, no suspend until alloc");
115 if (IsTLABAllocator(allocator)) {
116 byte_count = RoundUp(byte_count, space::BumpPointerSpace::kAlignment);
117 }
118 // If we have a thread local allocation we don't need to update bytes allocated.
119 if (IsTLABAllocator(allocator) && byte_count <= self->TlabSize()) {
120 obj = self->AllocTlab(byte_count);
121 DCHECK(obj != nullptr) << "AllocTlab can't fail";
122 obj->SetClass(klass);
123 if (kUseBakerReadBarrier) {
124 obj->AssertReadBarrierState();
125 }
126 bytes_allocated = byte_count;
127 usable_size = bytes_allocated;
128 no_suspend_pre_fence_visitor(obj, usable_size);
129 QuasiAtomic::ThreadFenceForConstructor();
130 } else if (
131 !kInstrumented && allocator == kAllocatorTypeRosAlloc &&
132 (obj = rosalloc_space_->AllocThreadLocal(self, byte_count, &bytes_allocated)) != nullptr &&
133 LIKELY(obj != nullptr)) {
134 DCHECK(!is_running_on_memory_tool_);
135 obj->SetClass(klass);
136 if (kUseBakerReadBarrier) {
137 obj->AssertReadBarrierState();
138 }
139 usable_size = bytes_allocated;
140 no_suspend_pre_fence_visitor(obj, usable_size);
141 QuasiAtomic::ThreadFenceForConstructor();
142 } else {
143 obj = TryToAllocate<kInstrumented, false>(self, allocator, byte_count, &bytes_allocated,
144 &usable_size, &bytes_tl_bulk_allocated);
145 if (UNLIKELY(obj == nullptr)) {
146 // AllocateInternalWithGc internally re-allows, and can cause, thread suspension, if
147 // someone instruments the entrypoints or changes the allocator in a suspend point here,
148 // we need to retry the allocation. It will send the pre-alloc event again.
149 obj = AllocateInternalWithGc(self,
150 allocator,
151 kInstrumented,
152 byte_count,
153 &bytes_allocated,
154 &usable_size,
155 &bytes_tl_bulk_allocated,
156 &klass);
157 if (obj == nullptr) {
158 // The only way that we can get a null return if there is no pending exception is if the
159 // allocator or instrumentation changed.
160 if (!self->IsExceptionPending()) {
161 // Since we are restarting, allow thread suspension.
162 ScopedAllowThreadSuspension ats;
163 // Get the new class size in case class redefinition changed the class size since alloc
164 // started.
165 int new_byte_count = klass->IsVariableSize()? byte_count : klass->GetObjectSize();
166 // AllocObject will pick up the new allocator type, and instrumented as true is the safe
167 // default.
168 return AllocObjectWithAllocator</*kInstrumented=*/true>(self,
169 klass,
170 new_byte_count,
171 GetUpdatedAllocator(allocator),
172 pre_fence_visitor);
173 }
174 return nullptr;
175 }
176 // Non-null result implies neither instrumentation nor allocator changed.
177 }
178 DCHECK_GT(bytes_allocated, 0u);
179 DCHECK_GT(usable_size, 0u);
180 obj->SetClass(klass);
181 if (kUseBakerReadBarrier) {
182 obj->AssertReadBarrierState();
183 }
184 if (collector::SemiSpace::kUseRememberedSet &&
185 UNLIKELY(allocator == kAllocatorTypeNonMoving)) {
186 // (Note this if statement will be constant folded away for the fast-path quick entry
187 // points.) Because SetClass() has no write barrier, the GC may need a write barrier in the
188 // case the object is non movable and points to a recently allocated movable class.
189 WriteBarrier::ForFieldWrite(obj, mirror::Object::ClassOffset(), klass);
190 }
191 no_suspend_pre_fence_visitor(obj, usable_size);
192 QuasiAtomic::ThreadFenceForConstructor();
193 }
194 if (bytes_tl_bulk_allocated > 0) {
195 starting_gc_num = GetCurrentGcNum();
196 size_t num_bytes_allocated_before =
197 num_bytes_allocated_.fetch_add(bytes_tl_bulk_allocated, std::memory_order_relaxed);
198 new_num_bytes_allocated = num_bytes_allocated_before + bytes_tl_bulk_allocated;
199 // Only trace when we get an increase in the number of bytes allocated. This happens when
200 // obtaining a new TLAB and isn't often enough to hurt performance according to golem.
201 if (region_space_) {
202 // With CC collector, during a GC cycle, the heap usage increases as
203 // there are two copies of evacuated objects. Therefore, add evac-bytes
204 // to the heap size. When the GC cycle is not running, evac-bytes
205 // are 0, as required.
206 TraceHeapSize(new_num_bytes_allocated + region_space_->EvacBytes());
207 } else {
208 TraceHeapSize(new_num_bytes_allocated);
209 }
210 // IsGcConcurrent() isn't known at compile time so we can optimize by not checking it for the
211 // BumpPointer or TLAB allocators. This is nice since it allows the entire if statement to be
212 // optimized out.
213 if (IsGcConcurrent() && UNLIKELY(ShouldConcurrentGCForJava(new_num_bytes_allocated))) {
214 need_gc = true;
215 }
216 GetMetrics()->TotalBytesAllocated()->Add(bytes_tl_bulk_allocated);
217 GetMetrics()->TotalBytesAllocatedDelta()->Add(bytes_tl_bulk_allocated);
218 }
219 }
220 if (kIsDebugBuild && Runtime::Current()->IsStarted()) {
221 CHECK_LE(obj->SizeOf(), usable_size);
222 }
223 // TODO: Deprecate.
224 if (kInstrumented) {
225 if (Runtime::Current()->HasStatsEnabled()) {
226 RuntimeStats* thread_stats = self->GetStats();
227 ++thread_stats->allocated_objects;
228 thread_stats->allocated_bytes += bytes_allocated;
229 RuntimeStats* global_stats = Runtime::Current()->GetStats();
230 ++global_stats->allocated_objects;
231 global_stats->allocated_bytes += bytes_allocated;
232 }
233 } else {
234 DCHECK(!Runtime::Current()->HasStatsEnabled());
235 }
236 if (kInstrumented) {
237 if (IsAllocTrackingEnabled()) {
238 // allocation_records_ is not null since it never becomes null after allocation tracking is
239 // enabled.
240 DCHECK(allocation_records_ != nullptr);
241 allocation_records_->RecordAllocation(self, &obj, bytes_allocated);
242 }
243 AllocationListener* l = alloc_listener_.load(std::memory_order_seq_cst);
244 if (l != nullptr) {
245 // Same as above. We assume that a listener that was once stored will never be deleted.
246 // Otherwise we'd have to perform this under a lock.
247 l->ObjectAllocated(self, &obj, bytes_allocated);
248 }
249 } else {
250 DCHECK(!IsAllocTrackingEnabled());
251 }
252 if (AllocatorHasAllocationStack(allocator)) {
253 PushOnAllocationStack(self, &obj);
254 }
255 if (kInstrumented) {
256 if (gc_stress_mode_) {
257 CheckGcStressMode(self, &obj);
258 }
259 } else {
260 DCHECK(!gc_stress_mode_);
261 }
262 if (need_gc) {
263 // Do this only once thread suspension is allowed again, and we're done with kInstrumented.
264 RequestConcurrentGCAndSaveObject(self, /*force_full=*/ false, starting_gc_num, &obj);
265 }
266 VerifyObject(obj);
267 self->VerifyStack();
268 return obj.Ptr();
269 }
270
271 // The size of a thread-local allocation stack in the number of references.
272 static constexpr size_t kThreadLocalAllocationStackSize = 128;
273
PushOnAllocationStack(Thread * self,ObjPtr<mirror::Object> * obj)274 inline void Heap::PushOnAllocationStack(Thread* self, ObjPtr<mirror::Object>* obj) {
275 if (kUseThreadLocalAllocationStack) {
276 if (UNLIKELY(!self->PushOnThreadLocalAllocationStack(obj->Ptr()))) {
277 PushOnThreadLocalAllocationStackWithInternalGC(self, obj);
278 }
279 } else if (UNLIKELY(!allocation_stack_->AtomicPushBack(obj->Ptr()))) {
280 PushOnAllocationStackWithInternalGC(self, obj);
281 }
282 }
283
284 template <bool kInstrumented, typename PreFenceVisitor>
AllocLargeObject(Thread * self,ObjPtr<mirror::Class> * klass,size_t byte_count,const PreFenceVisitor & pre_fence_visitor)285 inline mirror::Object* Heap::AllocLargeObject(Thread* self,
286 ObjPtr<mirror::Class>* klass,
287 size_t byte_count,
288 const PreFenceVisitor& pre_fence_visitor) {
289 // Save and restore the class in case it moves.
290 StackHandleScope<1> hs(self);
291 auto klass_wrapper = hs.NewHandleWrapper(klass);
292 mirror::Object* obj = AllocObjectWithAllocator<kInstrumented, false, PreFenceVisitor>
293 (self, *klass, byte_count, kAllocatorTypeLOS, pre_fence_visitor);
294 // Java Heap Profiler check and sample allocation.
295 JHPCheckNonTlabSampleAllocation(self, obj, byte_count);
296 return obj;
297 }
298
299 template <const bool kInstrumented, const bool kGrow>
TryToAllocate(Thread * self,AllocatorType allocator_type,size_t alloc_size,size_t * bytes_allocated,size_t * usable_size,size_t * bytes_tl_bulk_allocated)300 inline mirror::Object* Heap::TryToAllocate(Thread* self,
301 AllocatorType allocator_type,
302 size_t alloc_size,
303 size_t* bytes_allocated,
304 size_t* usable_size,
305 size_t* bytes_tl_bulk_allocated) {
306 if (allocator_type != kAllocatorTypeRegionTLAB &&
307 allocator_type != kAllocatorTypeTLAB &&
308 allocator_type != kAllocatorTypeRosAlloc &&
309 UNLIKELY(IsOutOfMemoryOnAllocation(allocator_type, alloc_size, kGrow))) {
310 return nullptr;
311 }
312 mirror::Object* ret;
313 switch (allocator_type) {
314 case kAllocatorTypeBumpPointer: {
315 DCHECK(bump_pointer_space_ != nullptr);
316 alloc_size = RoundUp(alloc_size, space::BumpPointerSpace::kAlignment);
317 ret = bump_pointer_space_->AllocNonvirtual(alloc_size);
318 if (LIKELY(ret != nullptr)) {
319 *bytes_allocated = alloc_size;
320 *usable_size = alloc_size;
321 *bytes_tl_bulk_allocated = alloc_size;
322 }
323 break;
324 }
325 case kAllocatorTypeRosAlloc: {
326 if (kInstrumented && UNLIKELY(is_running_on_memory_tool_)) {
327 // If running on ASan, we should be using the instrumented path.
328 size_t max_bytes_tl_bulk_allocated = rosalloc_space_->MaxBytesBulkAllocatedFor(alloc_size);
329 if (UNLIKELY(IsOutOfMemoryOnAllocation(allocator_type,
330 max_bytes_tl_bulk_allocated,
331 kGrow))) {
332 return nullptr;
333 }
334 ret = rosalloc_space_->Alloc(self, alloc_size, bytes_allocated, usable_size,
335 bytes_tl_bulk_allocated);
336 } else {
337 DCHECK(!is_running_on_memory_tool_);
338 size_t max_bytes_tl_bulk_allocated =
339 rosalloc_space_->MaxBytesBulkAllocatedForNonvirtual(alloc_size);
340 if (UNLIKELY(IsOutOfMemoryOnAllocation(allocator_type,
341 max_bytes_tl_bulk_allocated,
342 kGrow))) {
343 return nullptr;
344 }
345 if (!kInstrumented) {
346 DCHECK(!rosalloc_space_->CanAllocThreadLocal(self, alloc_size));
347 }
348 ret = rosalloc_space_->AllocNonvirtual(self,
349 alloc_size,
350 bytes_allocated,
351 usable_size,
352 bytes_tl_bulk_allocated);
353 }
354 break;
355 }
356 case kAllocatorTypeDlMalloc: {
357 if (kInstrumented && UNLIKELY(is_running_on_memory_tool_)) {
358 // If running on ASan, we should be using the instrumented path.
359 ret = dlmalloc_space_->Alloc(self,
360 alloc_size,
361 bytes_allocated,
362 usable_size,
363 bytes_tl_bulk_allocated);
364 } else {
365 DCHECK(!is_running_on_memory_tool_);
366 ret = dlmalloc_space_->AllocNonvirtual(self,
367 alloc_size,
368 bytes_allocated,
369 usable_size,
370 bytes_tl_bulk_allocated);
371 }
372 break;
373 }
374 case kAllocatorTypeNonMoving: {
375 ret = non_moving_space_->Alloc(self,
376 alloc_size,
377 bytes_allocated,
378 usable_size,
379 bytes_tl_bulk_allocated);
380 break;
381 }
382 case kAllocatorTypeLOS: {
383 ret = large_object_space_->Alloc(self,
384 alloc_size,
385 bytes_allocated,
386 usable_size,
387 bytes_tl_bulk_allocated);
388 // Note that the bump pointer spaces aren't necessarily next to
389 // the other continuous spaces like the non-moving alloc space or
390 // the zygote space.
391 DCHECK(ret == nullptr || large_object_space_->Contains(ret));
392 break;
393 }
394 case kAllocatorTypeRegion: {
395 DCHECK(region_space_ != nullptr);
396 alloc_size = RoundUp(alloc_size, space::RegionSpace::kAlignment);
397 ret = region_space_->AllocNonvirtual<false>(alloc_size,
398 bytes_allocated,
399 usable_size,
400 bytes_tl_bulk_allocated);
401 break;
402 }
403 case kAllocatorTypeTLAB:
404 FALLTHROUGH_INTENDED;
405 case kAllocatorTypeRegionTLAB: {
406 DCHECK_ALIGNED(alloc_size, kObjectAlignment);
407 static_assert(space::RegionSpace::kAlignment == space::BumpPointerSpace::kAlignment,
408 "mismatched alignments");
409 static_assert(kObjectAlignment == space::BumpPointerSpace::kAlignment,
410 "mismatched alignments");
411 if (UNLIKELY(self->TlabSize() < alloc_size)) {
412 return AllocWithNewTLAB(self,
413 allocator_type,
414 alloc_size,
415 kGrow,
416 bytes_allocated,
417 usable_size,
418 bytes_tl_bulk_allocated);
419 }
420 // The allocation can't fail.
421 ret = self->AllocTlab(alloc_size);
422 DCHECK(ret != nullptr);
423 *bytes_allocated = alloc_size;
424 *bytes_tl_bulk_allocated = 0; // Allocated in an existing buffer.
425 *usable_size = alloc_size;
426 break;
427 }
428 default: {
429 LOG(FATAL) << "Invalid allocator type";
430 ret = nullptr;
431 }
432 }
433 return ret;
434 }
435
ShouldAllocLargeObject(ObjPtr<mirror::Class> c,size_t byte_count)436 inline bool Heap::ShouldAllocLargeObject(ObjPtr<mirror::Class> c, size_t byte_count) const {
437 // We need to have a zygote space or else our newly allocated large object can end up in the
438 // Zygote resulting in it being prematurely freed.
439 // We can only do this for primitive objects since large objects will not be within the card table
440 // range. This also means that we rely on SetClass not dirtying the object's card.
441 return byte_count >= large_object_threshold_ && (c->IsPrimitiveArray() || c->IsStringClass());
442 }
443
IsOutOfMemoryOnAllocation(AllocatorType allocator_type ATTRIBUTE_UNUSED,size_t alloc_size,bool grow)444 inline bool Heap::IsOutOfMemoryOnAllocation(AllocatorType allocator_type ATTRIBUTE_UNUSED,
445 size_t alloc_size,
446 bool grow) {
447 size_t old_target = target_footprint_.load(std::memory_order_relaxed);
448 while (true) {
449 size_t old_allocated = num_bytes_allocated_.load(std::memory_order_relaxed);
450 size_t new_footprint = old_allocated + alloc_size;
451 // Tests against heap limits are inherently approximate, since multiple allocations may
452 // race, and this is not atomic with the allocation.
453 if (UNLIKELY(new_footprint <= old_target)) {
454 return false;
455 } else if (UNLIKELY(new_footprint > growth_limit_)) {
456 return true;
457 }
458 // We are between target_footprint_ and growth_limit_ .
459 if (IsGcConcurrent()) {
460 return false;
461 } else {
462 if (grow) {
463 if (target_footprint_.compare_exchange_weak(/*inout ref*/old_target, new_footprint,
464 std::memory_order_relaxed)) {
465 VlogHeapGrowth(old_target, new_footprint, alloc_size);
466 return false;
467 } // else try again.
468 } else {
469 return true;
470 }
471 }
472 }
473 }
474
ShouldConcurrentGCForJava(size_t new_num_bytes_allocated)475 inline bool Heap::ShouldConcurrentGCForJava(size_t new_num_bytes_allocated) {
476 // For a Java allocation, we only check whether the number of Java allocated bytes excceeds a
477 // threshold. By not considering native allocation here, we (a) ensure that Java heap bounds are
478 // maintained, and (b) reduce the cost of the check here.
479 return new_num_bytes_allocated >= concurrent_start_bytes_;
480 }
481
482 } // namespace gc
483 } // namespace art
484
485 #endif // ART_RUNTIME_GC_HEAP_INL_H_
486