1 /*
2 * Copyright (C) 2013 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #ifndef ART_RUNTIME_GC_HEAP_INL_H_
18 #define ART_RUNTIME_GC_HEAP_INL_H_
19
20 #include "heap.h"
21
22 #include "allocation_listener.h"
23 #include "base/quasi_atomic.h"
24 #include "base/time_utils.h"
25 #include "gc/accounting/atomic_stack.h"
26 #include "gc/accounting/card_table-inl.h"
27 #include "gc/allocation_record.h"
28 #include "gc/collector/semi_space.h"
29 #include "gc/space/bump_pointer_space-inl.h"
30 #include "gc/space/dlmalloc_space-inl.h"
31 #include "gc/space/large_object_space.h"
32 #include "gc/space/region_space-inl.h"
33 #include "gc/space/rosalloc_space-inl.h"
34 #include "handle_scope-inl.h"
35 #include "obj_ptr-inl.h"
36 #include "runtime.h"
37 #include "thread-inl.h"
38 #include "verify_object.h"
39 #include "write_barrier-inl.h"
40
41 namespace art {
42 namespace gc {
43
44 template <bool kInstrumented, bool kCheckLargeObject, typename PreFenceVisitor>
AllocObjectWithAllocator(Thread * self,ObjPtr<mirror::Class> klass,size_t byte_count,AllocatorType allocator,const PreFenceVisitor & pre_fence_visitor)45 inline mirror::Object* Heap::AllocObjectWithAllocator(Thread* self,
46 ObjPtr<mirror::Class> klass,
47 size_t byte_count,
48 AllocatorType allocator,
49 const PreFenceVisitor& pre_fence_visitor) {
50 if (kIsDebugBuild) {
51 CheckPreconditionsForAllocObject(klass, byte_count);
52 // Since allocation can cause a GC which will need to SuspendAll, make sure all allocations are
53 // done in the runnable state where suspension is expected.
54 CHECK_EQ(self->GetState(), kRunnable);
55 self->AssertThreadSuspensionIsAllowable();
56 self->AssertNoPendingException();
57 // Make sure to preserve klass.
58 StackHandleScope<1> hs(self);
59 HandleWrapperObjPtr<mirror::Class> h = hs.NewHandleWrapper(&klass);
60 self->PoisonObjectPointers();
61 }
62 // Need to check that we aren't the large object allocator since the large object allocation code
63 // path includes this function. If we didn't check we would have an infinite loop.
64 ObjPtr<mirror::Object> obj;
65 if (kCheckLargeObject && UNLIKELY(ShouldAllocLargeObject(klass, byte_count))) {
66 obj = AllocLargeObject<kInstrumented, PreFenceVisitor>(self, &klass, byte_count,
67 pre_fence_visitor);
68 if (obj != nullptr) {
69 return obj.Ptr();
70 } else {
71 // There should be an OOM exception, since we are retrying, clear it.
72 self->ClearException();
73 }
74 // If the large object allocation failed, try to use the normal spaces (main space,
75 // non moving space). This can happen if there is significant virtual address space
76 // fragmentation.
77 }
78 // bytes allocated for the (individual) object.
79 size_t bytes_allocated;
80 size_t usable_size;
81 size_t new_num_bytes_allocated = 0;
82 if (IsTLABAllocator(allocator)) {
83 byte_count = RoundUp(byte_count, space::BumpPointerSpace::kAlignment);
84 }
85 // If we have a thread local allocation we don't need to update bytes allocated.
86 if (IsTLABAllocator(allocator) && byte_count <= self->TlabSize()) {
87 obj = self->AllocTlab(byte_count);
88 DCHECK(obj != nullptr) << "AllocTlab can't fail";
89 obj->SetClass(klass);
90 if (kUseBakerReadBarrier) {
91 obj->AssertReadBarrierState();
92 }
93 bytes_allocated = byte_count;
94 usable_size = bytes_allocated;
95 pre_fence_visitor(obj, usable_size);
96 QuasiAtomic::ThreadFenceForConstructor();
97 } else if (
98 !kInstrumented && allocator == kAllocatorTypeRosAlloc &&
99 (obj = rosalloc_space_->AllocThreadLocal(self, byte_count, &bytes_allocated)) != nullptr &&
100 LIKELY(obj != nullptr)) {
101 DCHECK(!is_running_on_memory_tool_);
102 obj->SetClass(klass);
103 if (kUseBakerReadBarrier) {
104 obj->AssertReadBarrierState();
105 }
106 usable_size = bytes_allocated;
107 pre_fence_visitor(obj, usable_size);
108 QuasiAtomic::ThreadFenceForConstructor();
109 } else {
110 // Bytes allocated that includes bulk thread-local buffer allocations in addition to direct
111 // non-TLAB object allocations.
112 size_t bytes_tl_bulk_allocated = 0u;
113 obj = TryToAllocate<kInstrumented, false>(self, allocator, byte_count, &bytes_allocated,
114 &usable_size, &bytes_tl_bulk_allocated);
115 if (UNLIKELY(obj == nullptr)) {
116 // AllocateInternalWithGc can cause thread suspension, if someone instruments the entrypoints
117 // or changes the allocator in a suspend point here, we need to retry the allocation.
118 obj = AllocateInternalWithGc(self,
119 allocator,
120 kInstrumented,
121 byte_count,
122 &bytes_allocated,
123 &usable_size,
124 &bytes_tl_bulk_allocated, &klass);
125 if (obj == nullptr) {
126 // The only way that we can get a null return if there is no pending exception is if the
127 // allocator or instrumentation changed.
128 if (!self->IsExceptionPending()) {
129 // AllocObject will pick up the new allocator type, and instrumented as true is the safe
130 // default.
131 return AllocObject</*kInstrumented=*/true>(self,
132 klass,
133 byte_count,
134 pre_fence_visitor);
135 }
136 return nullptr;
137 }
138 }
139 DCHECK_GT(bytes_allocated, 0u);
140 DCHECK_GT(usable_size, 0u);
141 obj->SetClass(klass);
142 if (kUseBakerReadBarrier) {
143 obj->AssertReadBarrierState();
144 }
145 if (collector::SemiSpace::kUseRememberedSet && UNLIKELY(allocator == kAllocatorTypeNonMoving)) {
146 // (Note this if statement will be constant folded away for the
147 // fast-path quick entry points.) Because SetClass() has no write
148 // barrier, if a non-moving space allocation, we need a write
149 // barrier as the class pointer may point to the bump pointer
150 // space (where the class pointer is an "old-to-young" reference,
151 // though rare) under the GSS collector with the remembered set
152 // enabled. We don't need this for kAllocatorTypeRosAlloc/DlMalloc
153 // cases because we don't directly allocate into the main alloc
154 // space (besides promotions) under the SS/GSS collector.
155 WriteBarrier::ForFieldWrite(obj, mirror::Object::ClassOffset(), klass);
156 }
157 pre_fence_visitor(obj, usable_size);
158 QuasiAtomic::ThreadFenceForConstructor();
159 if (bytes_tl_bulk_allocated > 0) {
160 size_t num_bytes_allocated_before =
161 num_bytes_allocated_.fetch_add(bytes_tl_bulk_allocated, std::memory_order_relaxed);
162 new_num_bytes_allocated = num_bytes_allocated_before + bytes_tl_bulk_allocated;
163 // Only trace when we get an increase in the number of bytes allocated. This happens when
164 // obtaining a new TLAB and isn't often enough to hurt performance according to golem.
165 TraceHeapSize(new_num_bytes_allocated);
166 }
167 }
168 if (kIsDebugBuild && Runtime::Current()->IsStarted()) {
169 CHECK_LE(obj->SizeOf(), usable_size);
170 }
171 // TODO: Deprecate.
172 if (kInstrumented) {
173 if (Runtime::Current()->HasStatsEnabled()) {
174 RuntimeStats* thread_stats = self->GetStats();
175 ++thread_stats->allocated_objects;
176 thread_stats->allocated_bytes += bytes_allocated;
177 RuntimeStats* global_stats = Runtime::Current()->GetStats();
178 ++global_stats->allocated_objects;
179 global_stats->allocated_bytes += bytes_allocated;
180 }
181 } else {
182 DCHECK(!Runtime::Current()->HasStatsEnabled());
183 }
184 if (kInstrumented) {
185 if (IsAllocTrackingEnabled()) {
186 // allocation_records_ is not null since it never becomes null after allocation tracking is
187 // enabled.
188 DCHECK(allocation_records_ != nullptr);
189 allocation_records_->RecordAllocation(self, &obj, bytes_allocated);
190 }
191 AllocationListener* l = alloc_listener_.load(std::memory_order_seq_cst);
192 if (l != nullptr) {
193 // Same as above. We assume that a listener that was once stored will never be deleted.
194 // Otherwise we'd have to perform this under a lock.
195 l->ObjectAllocated(self, &obj, bytes_allocated);
196 }
197 } else {
198 DCHECK(!IsAllocTrackingEnabled());
199 }
200 if (AllocatorHasAllocationStack(allocator)) {
201 PushOnAllocationStack(self, &obj);
202 }
203 if (kInstrumented) {
204 if (gc_stress_mode_) {
205 CheckGcStressMode(self, &obj);
206 }
207 } else {
208 DCHECK(!gc_stress_mode_);
209 }
210 // IsGcConcurrent() isn't known at compile time so we can optimize by not checking it for
211 // the BumpPointer or TLAB allocators. This is nice since it allows the entire if statement to be
212 // optimized out. And for the other allocators, AllocatorMayHaveConcurrentGC is a constant since
213 // the allocator_type should be constant propagated.
214 if (AllocatorMayHaveConcurrentGC(allocator) && IsGcConcurrent()) {
215 // New_num_bytes_allocated is zero if we didn't update num_bytes_allocated_.
216 // That's fine.
217 CheckConcurrentGCForJava(self, new_num_bytes_allocated, &obj);
218 }
219 VerifyObject(obj);
220 self->VerifyStack();
221 return obj.Ptr();
222 }
223
224 // The size of a thread-local allocation stack in the number of references.
225 static constexpr size_t kThreadLocalAllocationStackSize = 128;
226
PushOnAllocationStack(Thread * self,ObjPtr<mirror::Object> * obj)227 inline void Heap::PushOnAllocationStack(Thread* self, ObjPtr<mirror::Object>* obj) {
228 if (kUseThreadLocalAllocationStack) {
229 if (UNLIKELY(!self->PushOnThreadLocalAllocationStack(obj->Ptr()))) {
230 PushOnThreadLocalAllocationStackWithInternalGC(self, obj);
231 }
232 } else if (UNLIKELY(!allocation_stack_->AtomicPushBack(obj->Ptr()))) {
233 PushOnAllocationStackWithInternalGC(self, obj);
234 }
235 }
236
237 template <bool kInstrumented, typename PreFenceVisitor>
AllocLargeObject(Thread * self,ObjPtr<mirror::Class> * klass,size_t byte_count,const PreFenceVisitor & pre_fence_visitor)238 inline mirror::Object* Heap::AllocLargeObject(Thread* self,
239 ObjPtr<mirror::Class>* klass,
240 size_t byte_count,
241 const PreFenceVisitor& pre_fence_visitor) {
242 // Save and restore the class in case it moves.
243 StackHandleScope<1> hs(self);
244 auto klass_wrapper = hs.NewHandleWrapper(klass);
245 return AllocObjectWithAllocator<kInstrumented, false, PreFenceVisitor>(self, *klass, byte_count,
246 kAllocatorTypeLOS,
247 pre_fence_visitor);
248 }
249
250 template <const bool kInstrumented, const bool kGrow>
TryToAllocate(Thread * self,AllocatorType allocator_type,size_t alloc_size,size_t * bytes_allocated,size_t * usable_size,size_t * bytes_tl_bulk_allocated)251 inline mirror::Object* Heap::TryToAllocate(Thread* self,
252 AllocatorType allocator_type,
253 size_t alloc_size,
254 size_t* bytes_allocated,
255 size_t* usable_size,
256 size_t* bytes_tl_bulk_allocated) {
257 if (allocator_type != kAllocatorTypeRegionTLAB &&
258 allocator_type != kAllocatorTypeTLAB &&
259 allocator_type != kAllocatorTypeRosAlloc &&
260 UNLIKELY(IsOutOfMemoryOnAllocation(allocator_type, alloc_size, kGrow))) {
261 return nullptr;
262 }
263 mirror::Object* ret;
264 switch (allocator_type) {
265 case kAllocatorTypeBumpPointer: {
266 DCHECK(bump_pointer_space_ != nullptr);
267 alloc_size = RoundUp(alloc_size, space::BumpPointerSpace::kAlignment);
268 ret = bump_pointer_space_->AllocNonvirtual(alloc_size);
269 if (LIKELY(ret != nullptr)) {
270 *bytes_allocated = alloc_size;
271 *usable_size = alloc_size;
272 *bytes_tl_bulk_allocated = alloc_size;
273 }
274 break;
275 }
276 case kAllocatorTypeRosAlloc: {
277 if (kInstrumented && UNLIKELY(is_running_on_memory_tool_)) {
278 // If running on ASan, we should be using the instrumented path.
279 size_t max_bytes_tl_bulk_allocated = rosalloc_space_->MaxBytesBulkAllocatedFor(alloc_size);
280 if (UNLIKELY(IsOutOfMemoryOnAllocation(allocator_type,
281 max_bytes_tl_bulk_allocated,
282 kGrow))) {
283 return nullptr;
284 }
285 ret = rosalloc_space_->Alloc(self, alloc_size, bytes_allocated, usable_size,
286 bytes_tl_bulk_allocated);
287 } else {
288 DCHECK(!is_running_on_memory_tool_);
289 size_t max_bytes_tl_bulk_allocated =
290 rosalloc_space_->MaxBytesBulkAllocatedForNonvirtual(alloc_size);
291 if (UNLIKELY(IsOutOfMemoryOnAllocation(allocator_type,
292 max_bytes_tl_bulk_allocated,
293 kGrow))) {
294 return nullptr;
295 }
296 if (!kInstrumented) {
297 DCHECK(!rosalloc_space_->CanAllocThreadLocal(self, alloc_size));
298 }
299 ret = rosalloc_space_->AllocNonvirtual(self,
300 alloc_size,
301 bytes_allocated,
302 usable_size,
303 bytes_tl_bulk_allocated);
304 }
305 break;
306 }
307 case kAllocatorTypeDlMalloc: {
308 if (kInstrumented && UNLIKELY(is_running_on_memory_tool_)) {
309 // If running on ASan, we should be using the instrumented path.
310 ret = dlmalloc_space_->Alloc(self,
311 alloc_size,
312 bytes_allocated,
313 usable_size,
314 bytes_tl_bulk_allocated);
315 } else {
316 DCHECK(!is_running_on_memory_tool_);
317 ret = dlmalloc_space_->AllocNonvirtual(self,
318 alloc_size,
319 bytes_allocated,
320 usable_size,
321 bytes_tl_bulk_allocated);
322 }
323 break;
324 }
325 case kAllocatorTypeNonMoving: {
326 ret = non_moving_space_->Alloc(self,
327 alloc_size,
328 bytes_allocated,
329 usable_size,
330 bytes_tl_bulk_allocated);
331 break;
332 }
333 case kAllocatorTypeLOS: {
334 ret = large_object_space_->Alloc(self,
335 alloc_size,
336 bytes_allocated,
337 usable_size,
338 bytes_tl_bulk_allocated);
339 // Note that the bump pointer spaces aren't necessarily next to
340 // the other continuous spaces like the non-moving alloc space or
341 // the zygote space.
342 DCHECK(ret == nullptr || large_object_space_->Contains(ret));
343 break;
344 }
345 case kAllocatorTypeRegion: {
346 DCHECK(region_space_ != nullptr);
347 alloc_size = RoundUp(alloc_size, space::RegionSpace::kAlignment);
348 ret = region_space_->AllocNonvirtual<false>(alloc_size,
349 bytes_allocated,
350 usable_size,
351 bytes_tl_bulk_allocated);
352 break;
353 }
354 case kAllocatorTypeTLAB:
355 FALLTHROUGH_INTENDED;
356 case kAllocatorTypeRegionTLAB: {
357 DCHECK_ALIGNED(alloc_size, kObjectAlignment);
358 static_assert(space::RegionSpace::kAlignment == space::BumpPointerSpace::kAlignment,
359 "mismatched alignments");
360 static_assert(kObjectAlignment == space::BumpPointerSpace::kAlignment,
361 "mismatched alignments");
362 if (UNLIKELY(self->TlabSize() < alloc_size)) {
363 // kAllocatorTypeTLAB may be the allocator for region space TLAB if the GC is not marking,
364 // that is why the allocator is not passed down.
365 return AllocWithNewTLAB(self,
366 alloc_size,
367 kGrow,
368 bytes_allocated,
369 usable_size,
370 bytes_tl_bulk_allocated);
371 }
372 // The allocation can't fail.
373 ret = self->AllocTlab(alloc_size);
374 DCHECK(ret != nullptr);
375 *bytes_allocated = alloc_size;
376 *bytes_tl_bulk_allocated = 0; // Allocated in an existing buffer.
377 *usable_size = alloc_size;
378 break;
379 }
380 default: {
381 LOG(FATAL) << "Invalid allocator type";
382 ret = nullptr;
383 }
384 }
385 return ret;
386 }
387
ShouldAllocLargeObject(ObjPtr<mirror::Class> c,size_t byte_count)388 inline bool Heap::ShouldAllocLargeObject(ObjPtr<mirror::Class> c, size_t byte_count) const {
389 // We need to have a zygote space or else our newly allocated large object can end up in the
390 // Zygote resulting in it being prematurely freed.
391 // We can only do this for primitive objects since large objects will not be within the card table
392 // range. This also means that we rely on SetClass not dirtying the object's card.
393 return byte_count >= large_object_threshold_ && (c->IsPrimitiveArray() || c->IsStringClass());
394 }
395
IsOutOfMemoryOnAllocation(AllocatorType allocator_type,size_t alloc_size,bool grow)396 inline bool Heap::IsOutOfMemoryOnAllocation(AllocatorType allocator_type,
397 size_t alloc_size,
398 bool grow) {
399 size_t old_target = target_footprint_.load(std::memory_order_relaxed);
400 while (true) {
401 size_t old_allocated = num_bytes_allocated_.load(std::memory_order_relaxed);
402 size_t new_footprint = old_allocated + alloc_size;
403 // Tests against heap limits are inherently approximate, since multiple allocations may
404 // race, and this is not atomic with the allocation.
405 if (UNLIKELY(new_footprint <= old_target)) {
406 return false;
407 } else if (UNLIKELY(new_footprint > growth_limit_)) {
408 return true;
409 }
410 // We are between target_footprint_ and growth_limit_ .
411 if (AllocatorMayHaveConcurrentGC(allocator_type) && IsGcConcurrent()) {
412 return false;
413 } else {
414 if (grow) {
415 if (target_footprint_.compare_exchange_weak(/*inout ref*/old_target, new_footprint,
416 std::memory_order_relaxed)) {
417 VlogHeapGrowth(old_target, new_footprint, alloc_size);
418 return false;
419 } // else try again.
420 } else {
421 return true;
422 }
423 }
424 }
425 }
426
ShouldConcurrentGCForJava(size_t new_num_bytes_allocated)427 inline bool Heap::ShouldConcurrentGCForJava(size_t new_num_bytes_allocated) {
428 // For a Java allocation, we only check whether the number of Java allocated bytes excceeds a
429 // threshold. By not considering native allocation here, we (a) ensure that Java heap bounds are
430 // maintained, and (b) reduce the cost of the check here.
431 return new_num_bytes_allocated >= concurrent_start_bytes_;
432 }
433
CheckConcurrentGCForJava(Thread * self,size_t new_num_bytes_allocated,ObjPtr<mirror::Object> * obj)434 inline void Heap::CheckConcurrentGCForJava(Thread* self,
435 size_t new_num_bytes_allocated,
436 ObjPtr<mirror::Object>* obj) {
437 if (UNLIKELY(ShouldConcurrentGCForJava(new_num_bytes_allocated))) {
438 RequestConcurrentGCAndSaveObject(self, false /* force_full */, obj);
439 }
440 }
441
442 } // namespace gc
443 } // namespace art
444
445 #endif // ART_RUNTIME_GC_HEAP_INL_H_
446