1 /* 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #ifndef ART_RUNTIME_GC_COLLECTOR_CONCURRENT_COPYING_H_ 18 #define ART_RUNTIME_GC_COLLECTOR_CONCURRENT_COPYING_H_ 19 20 #include "garbage_collector.h" 21 #include "gc/accounting/space_bitmap.h" 22 #include "immune_spaces.h" 23 #include "offsets.h" 24 25 #include <map> 26 #include <memory> 27 #include <unordered_map> 28 #include <vector> 29 30 namespace art { 31 class Barrier; 32 class Closure; 33 class RootInfo; 34 35 namespace mirror { 36 template<class MirrorType> class CompressedReference; 37 template<class MirrorType> class HeapReference; 38 class Object; 39 } // namespace mirror 40 41 namespace gc { 42 43 namespace accounting { 44 template<typename T> class AtomicStack; 45 typedef AtomicStack<mirror::Object> ObjectStack; 46 template <size_t kAlignment> class SpaceBitmap; 47 typedef SpaceBitmap<kObjectAlignment> ContinuousSpaceBitmap; 48 class HeapBitmap; 49 class ReadBarrierTable; 50 } // namespace accounting 51 52 namespace space { 53 class RegionSpace; 54 } // namespace space 55 56 namespace collector { 57 58 class ConcurrentCopying : public GarbageCollector { 59 public: 60 // Enable the no-from-space-refs verification at the pause. 61 static constexpr bool kEnableNoFromSpaceRefsVerification = kIsDebugBuild; 62 // Enable the from-space bytes/objects check. 63 static constexpr bool kEnableFromSpaceAccountingCheck = kIsDebugBuild; 64 // Enable verbose mode. 65 static constexpr bool kVerboseMode = false; 66 // If kGrayDirtyImmuneObjects is true then we gray dirty objects in the GC pause to prevent dirty 67 // pages. 68 static constexpr bool kGrayDirtyImmuneObjects = true; 69 70 ConcurrentCopying(Heap* heap, 71 bool young_gen, 72 bool use_generational_cc, 73 const std::string& name_prefix = "", 74 bool measure_read_barrier_slow_path = false); 75 ~ConcurrentCopying(); 76 77 void RunPhases() override 78 REQUIRES(!immune_gray_stack_lock_, 79 !mark_stack_lock_, 80 !rb_slow_path_histogram_lock_, 81 !skipped_blocks_lock_); 82 void InitializePhase() REQUIRES_SHARED(Locks::mutator_lock_) 83 REQUIRES(!mark_stack_lock_, !immune_gray_stack_lock_); 84 void MarkingPhase() REQUIRES_SHARED(Locks::mutator_lock_) 85 REQUIRES(!mark_stack_lock_); 86 void CopyingPhase() REQUIRES_SHARED(Locks::mutator_lock_) 87 REQUIRES(!mark_stack_lock_, !skipped_blocks_lock_, !immune_gray_stack_lock_); 88 void ReclaimPhase() REQUIRES_SHARED(Locks::mutator_lock_) REQUIRES(!mark_stack_lock_); 89 void FinishPhase() REQUIRES(!mark_stack_lock_, 90 !rb_slow_path_histogram_lock_, 91 !skipped_blocks_lock_); 92 93 void CaptureRssAtPeak() REQUIRES(!mark_stack_lock_); 94 void BindBitmaps() REQUIRES_SHARED(Locks::mutator_lock_) 95 REQUIRES(!Locks::heap_bitmap_lock_); GetGcType()96 GcType GetGcType() const override { 97 return (use_generational_cc_ && young_gen_) 98 ? kGcTypeSticky 99 : kGcTypePartial; 100 } GetCollectorType()101 CollectorType GetCollectorType() const override { 102 return kCollectorTypeCC; 103 } 104 void RevokeAllThreadLocalBuffers() override; 105 // Creates inter-region ref bitmaps for region-space and non-moving-space. 106 // Gets called in Heap construction after the two spaces are created. 107 void CreateInterRegionRefBitmaps(); SetRegionSpace(space::RegionSpace * region_space)108 void SetRegionSpace(space::RegionSpace* region_space) { 109 DCHECK(region_space != nullptr); 110 region_space_ = region_space; 111 } RegionSpace()112 space::RegionSpace* RegionSpace() { 113 return region_space_; 114 } 115 // Assert the to-space invariant for a heap reference `ref` held in `obj` at offset `offset`. 116 void AssertToSpaceInvariant(mirror::Object* obj, MemberOffset offset, mirror::Object* ref) 117 REQUIRES_SHARED(Locks::mutator_lock_); 118 // Assert the to-space invariant for a GC root reference `ref`. 119 void AssertToSpaceInvariant(GcRootSource* gc_root_source, mirror::Object* ref) 120 REQUIRES_SHARED(Locks::mutator_lock_); IsInToSpace(mirror::Object * ref)121 bool IsInToSpace(mirror::Object* ref) REQUIRES_SHARED(Locks::mutator_lock_) { 122 DCHECK(ref != nullptr); 123 return IsMarked(ref) == ref; 124 } 125 // Mark object `from_ref`, copying it to the to-space if needed. 126 template<bool kGrayImmuneObject = true, bool kNoUnEvac = false, bool kFromGCThread = false> 127 ALWAYS_INLINE mirror::Object* Mark(Thread* const self, 128 mirror::Object* from_ref, 129 mirror::Object* holder = nullptr, 130 MemberOffset offset = MemberOffset(0)) 131 REQUIRES_SHARED(Locks::mutator_lock_) 132 REQUIRES(!mark_stack_lock_, !skipped_blocks_lock_, !immune_gray_stack_lock_); 133 ALWAYS_INLINE mirror::Object* MarkFromReadBarrier(mirror::Object* from_ref) 134 REQUIRES_SHARED(Locks::mutator_lock_) 135 REQUIRES(!mark_stack_lock_, !skipped_blocks_lock_, !immune_gray_stack_lock_); IsMarking()136 bool IsMarking() const { 137 return is_marking_; 138 } 139 // We may want to use read barrier entrypoints before is_marking_ is true since concurrent graying 140 // creates a small window where we might dispatch on these entrypoints. IsUsingReadBarrierEntrypoints()141 bool IsUsingReadBarrierEntrypoints() const { 142 return is_using_read_barrier_entrypoints_; 143 } IsActive()144 bool IsActive() const { 145 return is_active_; 146 } GetBarrier()147 Barrier& GetBarrier() { 148 return *gc_barrier_; 149 } IsWeakRefAccessEnabled()150 bool IsWeakRefAccessEnabled() REQUIRES(Locks::thread_list_lock_) { 151 return weak_ref_access_enabled_; 152 } 153 void RevokeThreadLocalMarkStack(Thread* thread) REQUIRES(!mark_stack_lock_); 154 155 // Blindly return the forwarding pointer from the lockword, or null if there is none. 156 static mirror::Object* GetFwdPtrUnchecked(mirror::Object* from_ref) 157 REQUIRES_SHARED(Locks::mutator_lock_); 158 159 // If marked, return the to-space object, otherwise null. 160 mirror::Object* IsMarked(mirror::Object* from_ref) override 161 REQUIRES_SHARED(Locks::mutator_lock_); 162 163 void AssertNoThreadMarkStackMapping(Thread* thread) REQUIRES(!mark_stack_lock_); 164 165 private: 166 void PushOntoMarkStack(Thread* const self, mirror::Object* obj) 167 REQUIRES_SHARED(Locks::mutator_lock_) 168 REQUIRES(!mark_stack_lock_); 169 // Returns a to-space copy of the from-space object from_ref, and atomically installs a 170 // forwarding pointer. Ensures that the forwarding reference is visible to other threads before 171 // the returned to-space pointer becomes visible to them. 172 mirror::Object* Copy(Thread* const self, 173 mirror::Object* from_ref, 174 mirror::Object* holder, 175 MemberOffset offset) 176 REQUIRES_SHARED(Locks::mutator_lock_) 177 REQUIRES(!mark_stack_lock_, !skipped_blocks_lock_, !immune_gray_stack_lock_); 178 // Scan the reference fields of object `to_ref`. 179 template <bool kNoUnEvac> 180 void Scan(mirror::Object* to_ref, size_t obj_size = 0) REQUIRES_SHARED(Locks::mutator_lock_) 181 REQUIRES(!mark_stack_lock_); 182 // Scan the reference fields of object 'obj' in the dirty cards during 183 // card-table scan. In addition to visiting the references, it also sets the 184 // read-barrier state to gray for Reference-type objects to ensure that 185 // GetReferent() called on these objects calls the read-barrier on the referent. 186 template <bool kNoUnEvac> 187 void ScanDirtyObject(mirror::Object* obj) REQUIRES_SHARED(Locks::mutator_lock_) 188 REQUIRES(!mark_stack_lock_); 189 // Process a field. 190 template <bool kNoUnEvac> 191 void Process(mirror::Object* obj, MemberOffset offset) 192 REQUIRES_SHARED(Locks::mutator_lock_) 193 REQUIRES(!mark_stack_lock_ , !skipped_blocks_lock_, !immune_gray_stack_lock_); 194 void VisitRoots(mirror::Object*** roots, size_t count, const RootInfo& info) override 195 REQUIRES_SHARED(Locks::mutator_lock_) 196 REQUIRES(!mark_stack_lock_, !skipped_blocks_lock_, !immune_gray_stack_lock_); 197 template<bool kGrayImmuneObject> 198 void MarkRoot(Thread* const self, mirror::CompressedReference<mirror::Object>* root) 199 REQUIRES_SHARED(Locks::mutator_lock_) 200 REQUIRES(!mark_stack_lock_, !skipped_blocks_lock_, !immune_gray_stack_lock_); 201 void VisitRoots(mirror::CompressedReference<mirror::Object>** roots, 202 size_t count, 203 const RootInfo& info) override 204 REQUIRES_SHARED(Locks::mutator_lock_) 205 REQUIRES(!mark_stack_lock_, !skipped_blocks_lock_, !immune_gray_stack_lock_); 206 void VerifyNoFromSpaceReferences() REQUIRES(Locks::mutator_lock_); 207 accounting::ObjectStack* GetAllocationStack(); 208 accounting::ObjectStack* GetLiveStack(); 209 void ProcessMarkStack() override REQUIRES_SHARED(Locks::mutator_lock_) 210 REQUIRES(!mark_stack_lock_); 211 bool ProcessMarkStackOnce() REQUIRES_SHARED(Locks::mutator_lock_) REQUIRES(!mark_stack_lock_); 212 void ProcessMarkStackRef(mirror::Object* to_ref) REQUIRES_SHARED(Locks::mutator_lock_) 213 REQUIRES(!mark_stack_lock_); 214 void GrayAllDirtyImmuneObjects() 215 REQUIRES(Locks::mutator_lock_) 216 REQUIRES(!mark_stack_lock_); 217 void GrayAllNewlyDirtyImmuneObjects() 218 REQUIRES(Locks::mutator_lock_) 219 REQUIRES(!mark_stack_lock_); 220 void VerifyGrayImmuneObjects() 221 REQUIRES(Locks::mutator_lock_) 222 REQUIRES(!mark_stack_lock_); 223 void VerifyNoMissingCardMarks() 224 REQUIRES(Locks::mutator_lock_) 225 REQUIRES(!mark_stack_lock_); 226 template <typename Processor> 227 size_t ProcessThreadLocalMarkStacks(bool disable_weak_ref_access, 228 Closure* checkpoint_callback, 229 const Processor& processor) 230 REQUIRES_SHARED(Locks::mutator_lock_) REQUIRES(!mark_stack_lock_); 231 void RevokeThreadLocalMarkStacks(bool disable_weak_ref_access, Closure* checkpoint_callback) 232 REQUIRES_SHARED(Locks::mutator_lock_); 233 void SwitchToSharedMarkStackMode() REQUIRES_SHARED(Locks::mutator_lock_) 234 REQUIRES(!mark_stack_lock_); 235 void SwitchToGcExclusiveMarkStackMode() REQUIRES_SHARED(Locks::mutator_lock_); 236 void DelayReferenceReferent(ObjPtr<mirror::Class> klass, 237 ObjPtr<mirror::Reference> reference) override 238 REQUIRES_SHARED(Locks::mutator_lock_); 239 void ProcessReferences(Thread* self) REQUIRES_SHARED(Locks::mutator_lock_); 240 mirror::Object* MarkObject(mirror::Object* from_ref) override 241 REQUIRES_SHARED(Locks::mutator_lock_) 242 REQUIRES(!mark_stack_lock_, !skipped_blocks_lock_, !immune_gray_stack_lock_); 243 void MarkHeapReference(mirror::HeapReference<mirror::Object>* from_ref, 244 bool do_atomic_update) override 245 REQUIRES_SHARED(Locks::mutator_lock_) 246 REQUIRES(!mark_stack_lock_, !skipped_blocks_lock_, !immune_gray_stack_lock_); 247 bool IsMarkedInUnevacFromSpace(mirror::Object* from_ref) 248 REQUIRES_SHARED(Locks::mutator_lock_); 249 bool IsMarkedInNonMovingSpace(mirror::Object* from_ref) 250 REQUIRES_SHARED(Locks::mutator_lock_); 251 bool IsNullOrMarkedHeapReference(mirror::HeapReference<mirror::Object>* field, 252 bool do_atomic_update) override 253 REQUIRES_SHARED(Locks::mutator_lock_); 254 void SweepSystemWeaks(Thread* self) 255 REQUIRES_SHARED(Locks::mutator_lock_) REQUIRES(!Locks::heap_bitmap_lock_); 256 // Sweep unmarked objects to complete the garbage collection. Full GCs sweep 257 // all allocation spaces (except the region space). Sticky-bit GCs just sweep 258 // a subset of the heap. 259 void Sweep(bool swap_bitmaps) 260 REQUIRES_SHARED(Locks::mutator_lock_) REQUIRES(Locks::heap_bitmap_lock_, !mark_stack_lock_); 261 // Sweep only pointers within an array. 262 void SweepArray(accounting::ObjectStack* allocation_stack_, bool swap_bitmaps) 263 REQUIRES_SHARED(Locks::mutator_lock_) REQUIRES(Locks::heap_bitmap_lock_, !mark_stack_lock_); 264 void SweepLargeObjects(bool swap_bitmaps) 265 REQUIRES_SHARED(Locks::mutator_lock_) REQUIRES(Locks::heap_bitmap_lock_); 266 void MarkZygoteLargeObjects() 267 REQUIRES_SHARED(Locks::mutator_lock_); 268 void FillWithFakeObject(Thread* const self, mirror::Object* fake_obj, size_t byte_size) 269 REQUIRES(!mark_stack_lock_, !skipped_blocks_lock_, !immune_gray_stack_lock_) 270 REQUIRES_SHARED(Locks::mutator_lock_); 271 mirror::Object* AllocateInSkippedBlock(Thread* const self, size_t alloc_size) 272 REQUIRES(!mark_stack_lock_, !skipped_blocks_lock_, !immune_gray_stack_lock_) 273 REQUIRES_SHARED(Locks::mutator_lock_); 274 void CheckEmptyMarkStack() REQUIRES_SHARED(Locks::mutator_lock_) REQUIRES(!mark_stack_lock_); 275 void IssueEmptyCheckpoint() REQUIRES_SHARED(Locks::mutator_lock_); 276 bool IsOnAllocStack(mirror::Object* ref) REQUIRES_SHARED(Locks::mutator_lock_); 277 // Return the forwarding pointer from the lockword. The argument must be in from space. 278 mirror::Object* GetFwdPtr(mirror::Object* from_ref) REQUIRES_SHARED(Locks::mutator_lock_); 279 void FlipThreadRoots() REQUIRES(!Locks::mutator_lock_); 280 void SwapStacks() REQUIRES_SHARED(Locks::mutator_lock_); 281 void RecordLiveStackFreezeSize(Thread* self); 282 void ComputeUnevacFromSpaceLiveRatio(); 283 void LogFromSpaceRefHolder(mirror::Object* obj, MemberOffset offset) 284 REQUIRES_SHARED(Locks::mutator_lock_); 285 // Dump information about reference `ref` and return it as a string. 286 // Use `ref_name` to name the reference in messages. Each message is prefixed with `indent`. 287 std::string DumpReferenceInfo(mirror::Object* ref, const char* ref_name, const char* indent = "") 288 REQUIRES_SHARED(Locks::mutator_lock_); 289 // Dump information about heap reference `ref`, referenced from object `obj` at offset `offset`, 290 // and return it as a string. 291 std::string DumpHeapReference(mirror::Object* obj, MemberOffset offset, mirror::Object* ref) 292 REQUIRES_SHARED(Locks::mutator_lock_); 293 // Dump information about GC root `ref` and return it as a string. 294 std::string DumpGcRoot(mirror::Object* ref) REQUIRES_SHARED(Locks::mutator_lock_); 295 void AssertToSpaceInvariantInNonMovingSpace(mirror::Object* obj, mirror::Object* ref) 296 REQUIRES_SHARED(Locks::mutator_lock_); 297 void ReenableWeakRefAccess(Thread* self) REQUIRES_SHARED(Locks::mutator_lock_); 298 void DisableMarking() REQUIRES_SHARED(Locks::mutator_lock_); 299 void IssueDisableMarkingCheckpoint() REQUIRES_SHARED(Locks::mutator_lock_); 300 void ExpandGcMarkStack() REQUIRES_SHARED(Locks::mutator_lock_); 301 mirror::Object* MarkNonMoving(Thread* const self, 302 mirror::Object* from_ref, 303 mirror::Object* holder = nullptr, 304 MemberOffset offset = MemberOffset(0)) 305 REQUIRES_SHARED(Locks::mutator_lock_) 306 REQUIRES(!mark_stack_lock_, !skipped_blocks_lock_); 307 ALWAYS_INLINE mirror::Object* MarkUnevacFromSpaceRegion(Thread* const self, 308 mirror::Object* from_ref, 309 accounting::SpaceBitmap<kObjectAlignment>* bitmap) 310 REQUIRES_SHARED(Locks::mutator_lock_) 311 REQUIRES(!mark_stack_lock_, !skipped_blocks_lock_); 312 template<bool kGrayImmuneObject> 313 ALWAYS_INLINE mirror::Object* MarkImmuneSpace(Thread* const self, 314 mirror::Object* from_ref) 315 REQUIRES_SHARED(Locks::mutator_lock_) REQUIRES(!immune_gray_stack_lock_); 316 void ScanImmuneObject(mirror::Object* obj) 317 REQUIRES_SHARED(Locks::mutator_lock_) REQUIRES(!mark_stack_lock_); 318 mirror::Object* MarkFromReadBarrierWithMeasurements(Thread* const self, 319 mirror::Object* from_ref) 320 REQUIRES_SHARED(Locks::mutator_lock_) 321 REQUIRES(!mark_stack_lock_, !skipped_blocks_lock_, !immune_gray_stack_lock_); 322 void DumpPerformanceInfo(std::ostream& os) override REQUIRES(!rb_slow_path_histogram_lock_); 323 // Set the read barrier mark entrypoints to non-null. 324 void ActivateReadBarrierEntrypoints(); 325 326 void CaptureThreadRootsForMarking() REQUIRES_SHARED(Locks::mutator_lock_); 327 void AddLiveBytesAndScanRef(mirror::Object* ref) REQUIRES_SHARED(Locks::mutator_lock_); 328 bool TestMarkBitmapForRef(mirror::Object* ref) REQUIRES_SHARED(Locks::mutator_lock_); 329 template <bool kAtomic = false> 330 bool TestAndSetMarkBitForRef(mirror::Object* ref) REQUIRES_SHARED(Locks::mutator_lock_); 331 void PushOntoLocalMarkStack(mirror::Object* ref) REQUIRES_SHARED(Locks::mutator_lock_); 332 void ProcessMarkStackForMarkingAndComputeLiveBytes() REQUIRES_SHARED(Locks::mutator_lock_) 333 REQUIRES(!mark_stack_lock_); 334 335 void RemoveThreadMarkStackMapping(Thread* thread, accounting::ObjectStack* tl_mark_stack) 336 REQUIRES(mark_stack_lock_); 337 void AddThreadMarkStackMapping(Thread* thread, accounting::ObjectStack* tl_mark_stack) 338 REQUIRES(mark_stack_lock_); 339 void AssertEmptyThreadMarkStackMap() REQUIRES(mark_stack_lock_); 340 341 space::RegionSpace* region_space_; // The underlying region space. 342 std::unique_ptr<Barrier> gc_barrier_; 343 std::unique_ptr<accounting::ObjectStack> gc_mark_stack_; 344 345 // If true, enable generational collection when using the Concurrent Copying 346 // (CC) collector, i.e. use sticky-bit CC for minor collections and (full) CC 347 // for major collections. Generational CC collection is currently only 348 // compatible with Baker read barriers. Set in Heap constructor. 349 const bool use_generational_cc_; 350 351 // Generational "sticky", only trace through dirty objects in region space. 352 const bool young_gen_; 353 354 // If true, the GC thread is done scanning marked objects on dirty and aged 355 // card (see ConcurrentCopying::CopyingPhase). 356 Atomic<bool> done_scanning_; 357 358 // The read-barrier mark-bit stack. Stores object references whose 359 // mark bit has been set by ConcurrentCopying::MarkFromReadBarrier, 360 // so that this bit can be reset at the end of the collection in 361 // ConcurrentCopying::FinishPhase. The mark bit of an object can be 362 // used by mutator read barrier code to quickly test whether that 363 // object has been already marked. 364 std::unique_ptr<accounting::ObjectStack> rb_mark_bit_stack_; 365 // Thread-unsafe Boolean value hinting that `rb_mark_bit_stack_` is 366 // full. A thread-safe test of whether the read-barrier mark-bit 367 // stack is full is implemented by `rb_mark_bit_stack_->AtomicPushBack(ref)` 368 // (see use case in ConcurrentCopying::MarkFromReadBarrier). 369 bool rb_mark_bit_stack_full_; 370 371 // Guards access to pooled_mark_stacks_ and revoked_mark_stacks_ vectors. 372 // Also guards destruction and revocations of thread-local mark-stacks. 373 // Clearing thread-local mark-stack (by other threads or during destruction) 374 // should be guarded by it. 375 Mutex mark_stack_lock_ DEFAULT_MUTEX_ACQUIRED_AFTER; 376 std::vector<accounting::ObjectStack*> revoked_mark_stacks_ 377 GUARDED_BY(mark_stack_lock_); 378 static constexpr size_t kMarkStackSize = kPageSize; 379 static constexpr size_t kMarkStackPoolSize = 256; 380 std::vector<accounting::ObjectStack*> pooled_mark_stacks_ 381 GUARDED_BY(mark_stack_lock_); 382 Thread* thread_running_gc_; 383 bool is_marking_; // True while marking is ongoing. 384 // True while we might dispatch on the read barrier entrypoints. 385 bool is_using_read_barrier_entrypoints_; 386 bool is_active_; // True while the collection is ongoing. 387 bool is_asserting_to_space_invariant_; // True while asserting the to-space invariant. 388 ImmuneSpaces immune_spaces_; 389 accounting::ContinuousSpaceBitmap* region_space_bitmap_; 390 // A cache of Heap::GetMarkBitmap(). 391 accounting::HeapBitmap* heap_mark_bitmap_; 392 size_t live_stack_freeze_size_; 393 size_t from_space_num_objects_at_first_pause_; // Computed if kEnableFromSpaceAccountingCheck 394 size_t from_space_num_bytes_at_first_pause_; // Computed if kEnableFromSpaceAccountingCheck 395 Atomic<int> is_mark_stack_push_disallowed_; 396 enum MarkStackMode { 397 kMarkStackModeOff = 0, // Mark stack is off. 398 kMarkStackModeThreadLocal, // All threads except for the GC-running thread push refs onto 399 // thread-local mark stacks. The GC-running thread pushes onto and 400 // pops off the GC mark stack without a lock. 401 kMarkStackModeShared, // All threads share the GC mark stack with a lock. 402 kMarkStackModeGcExclusive // The GC-running thread pushes onto and pops from the GC mark stack 403 // without a lock. Other threads won't access the mark stack. 404 }; 405 Atomic<MarkStackMode> mark_stack_mode_; 406 bool weak_ref_access_enabled_ GUARDED_BY(Locks::thread_list_lock_); 407 408 // How many objects and bytes we moved. The GC thread moves many more objects 409 // than mutators. Therefore, we separate the two to avoid CAS. Bytes_moved_ and 410 // bytes_moved_gc_thread_ are critical for GC triggering; the others are just informative. 411 Atomic<size_t> bytes_moved_; // Used by mutators 412 Atomic<size_t> objects_moved_; // Used by mutators 413 414 // copied_live_bytes_ratio_sum_ is read and written by CC per GC, in 415 // ReclaimPhase, and is read by DumpPerformanceInfo (potentially from another 416 // thread). However, at present, DumpPerformanceInfo is only called when the 417 // runtime shuts down, so no concurrent access. The same reasoning goes for 418 // gc_count_ and reclaimed_bytes_ratio_sum_ 419 420 // The sum of of all copied live bytes ratio (to_bytes/from_bytes) 421 float copied_live_bytes_ratio_sum_; 422 // The number of GC counts, used to calculate the average above. (It doesn't 423 // include GC where from_bytes is zero, IOW, from-space is empty, which is 424 // possible for minor GC if all allocated objects are in non-moving 425 // space.) 426 size_t gc_count_; 427 // Bit is set if the corresponding object has inter-region references that 428 // were found during the marking phase of two-phase full-heap GC cycle. 429 accounting::ContinuousSpaceBitmap region_space_inter_region_bitmap_; 430 accounting::ContinuousSpaceBitmap non_moving_space_inter_region_bitmap_; 431 432 // reclaimed_bytes_ratio = reclaimed_bytes/num_allocated_bytes per GC cycle 433 float reclaimed_bytes_ratio_sum_; 434 435 // Used only by GC thread, so need not be atomic. Also, should be kept 436 // in a different cacheline than bytes/objects_moved_ (above) to avoid false 437 // cacheline sharing. 438 size_t bytes_moved_gc_thread_; 439 size_t objects_moved_gc_thread_; 440 uint64_t bytes_scanned_; 441 uint64_t cumulative_bytes_moved_; 442 uint64_t cumulative_objects_moved_; 443 444 // The skipped blocks are memory blocks/chucks that were copies of 445 // objects that were unused due to lost races (cas failures) at 446 // object copy/forward pointer install. They may be reused. 447 // Skipped blocks are always in region space. Their size is included directly 448 // in num_bytes_allocated_, i.e. they are treated as allocated, but may be directly 449 // used without going through a GC cycle like other objects. They are reused only 450 // if we run out of region space. TODO: Revisit this design. 451 Mutex skipped_blocks_lock_ DEFAULT_MUTEX_ACQUIRED_AFTER; 452 std::multimap<size_t, uint8_t*> skipped_blocks_map_ GUARDED_BY(skipped_blocks_lock_); 453 Atomic<size_t> to_space_bytes_skipped_; 454 Atomic<size_t> to_space_objects_skipped_; 455 456 // If measure_read_barrier_slow_path_ is true, we count how long is spent in MarkFromReadBarrier 457 // and also log. 458 bool measure_read_barrier_slow_path_; 459 // mark_from_read_barrier_measurements_ is true if systrace is enabled or 460 // measure_read_barrier_time_ is true. 461 bool mark_from_read_barrier_measurements_; 462 Atomic<uint64_t> rb_slow_path_ns_; 463 Atomic<uint64_t> rb_slow_path_count_; 464 Atomic<uint64_t> rb_slow_path_count_gc_; 465 mutable Mutex rb_slow_path_histogram_lock_ DEFAULT_MUTEX_ACQUIRED_AFTER; 466 Histogram<uint64_t> rb_slow_path_time_histogram_ GUARDED_BY(rb_slow_path_histogram_lock_); 467 uint64_t rb_slow_path_count_total_ GUARDED_BY(rb_slow_path_histogram_lock_); 468 uint64_t rb_slow_path_count_gc_total_ GUARDED_BY(rb_slow_path_histogram_lock_); 469 470 accounting::ReadBarrierTable* rb_table_; 471 bool force_evacuate_all_; // True if all regions are evacuated. 472 Atomic<bool> updated_all_immune_objects_; 473 bool gc_grays_immune_objects_; 474 Mutex immune_gray_stack_lock_ DEFAULT_MUTEX_ACQUIRED_AFTER; 475 std::vector<mirror::Object*> immune_gray_stack_ GUARDED_BY(immune_gray_stack_lock_); 476 477 // Class of java.lang.Object. Filled in from WellKnownClasses in FlipCallback. Must 478 // be filled in before flipping thread roots so that FillWithFakeObject can run. Not 479 // ObjPtr since the GC may transition to suspended and runnable between phases. 480 mirror::Class* java_lang_Object_; 481 482 // Sweep array free buffer, used to sweep the spaces based on an array more 483 // efficiently, by recording dead objects to be freed in batches (see 484 // ConcurrentCopying::SweepArray). 485 MemMap sweep_array_free_buffer_mem_map_; 486 487 // Use signed because after_gc may be larger than before_gc. 488 int64_t num_bytes_allocated_before_gc_; 489 490 class ActivateReadBarrierEntrypointsCallback; 491 class ActivateReadBarrierEntrypointsCheckpoint; 492 class AssertToSpaceInvariantFieldVisitor; 493 class AssertToSpaceInvariantRefsVisitor; 494 class ClearBlackPtrsVisitor; 495 class ComputeUnevacFromSpaceLiveRatioVisitor; 496 class DisableMarkingCallback; 497 class DisableMarkingCheckpoint; 498 class DisableWeakRefAccessCallback; 499 class FlipCallback; 500 template <bool kConcurrent> class GrayImmuneObjectVisitor; 501 class ImmuneSpaceScanObjVisitor; 502 class LostCopyVisitor; 503 template <bool kNoUnEvac> class RefFieldsVisitor; 504 class RevokeThreadLocalMarkStackCheckpoint; 505 class ScopedGcGraysImmuneObjects; 506 class ThreadFlipVisitor; 507 class VerifyGrayImmuneObjectsVisitor; 508 class VerifyNoFromSpaceRefsFieldVisitor; 509 class VerifyNoFromSpaceRefsVisitor; 510 class VerifyNoMissingCardMarkVisitor; 511 class ImmuneSpaceCaptureRefsVisitor; 512 template <bool kAtomicTestAndSet = false> class CaptureRootsForMarkingVisitor; 513 class CaptureThreadRootsForMarkingAndCheckpoint; 514 template <bool kHandleInterRegionRefs> class ComputeLiveBytesAndMarkRefFieldsVisitor; 515 516 DISALLOW_IMPLICIT_CONSTRUCTORS(ConcurrentCopying); 517 }; 518 519 } // namespace collector 520 } // namespace gc 521 } // namespace art 522 523 #endif // ART_RUNTIME_GC_COLLECTOR_CONCURRENT_COPYING_H_ 524