1 /*
2 * Copyright (C) 2013 Google Inc. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions are
6 * met:
7 *
8 * * Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * * Redistributions in binary form must reproduce the above
11 * copyright notice, this list of conditions and the following disclaimer
12 * in the documentation and/or other materials provided with the
13 * distribution.
14 * * Neither the name of Google Inc. nor the names of its
15 * contributors may be used to endorse or promote products derived from
16 * this software without specific prior written permission.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 */
30
31 #include "config.h"
32 #include "platform/heap/ThreadState.h"
33
34 #include "platform/ScriptForbiddenScope.h"
35 #include "platform/TraceEvent.h"
36 #include "platform/heap/AddressSanitizer.h"
37 #include "platform/heap/CallbackStack.h"
38 #include "platform/heap/Handle.h"
39 #include "platform/heap/Heap.h"
40 #include "public/platform/Platform.h"
41 #include "public/platform/WebThread.h"
42 #include "wtf/ThreadingPrimitives.h"
43 #if ENABLE(GC_PROFILE_HEAP)
44 #include "platform/TracedValue.h"
45 #endif
46
47 #if OS(WIN)
48 #include <stddef.h>
49 #include <windows.h>
50 #include <winnt.h>
51 #elif defined(__GLIBC__)
52 extern "C" void* __libc_stack_end; // NOLINT
53 #endif
54
55 #if defined(MEMORY_SANITIZER)
56 #include <sanitizer/msan_interface.h>
57 #endif
58
59 namespace blink {
60
getStackStart()61 static void* getStackStart()
62 {
63 #if defined(__GLIBC__) || OS(ANDROID)
64 pthread_attr_t attr;
65 if (!pthread_getattr_np(pthread_self(), &attr)) {
66 void* base;
67 size_t size;
68 int error = pthread_attr_getstack(&attr, &base, &size);
69 RELEASE_ASSERT(!error);
70 pthread_attr_destroy(&attr);
71 return reinterpret_cast<Address>(base) + size;
72 }
73 #if defined(__GLIBC__)
74 // pthread_getattr_np can fail for the main thread. In this case
75 // just like NaCl we rely on the __libc_stack_end to give us
76 // the start of the stack.
77 // See https://code.google.com/p/nativeclient/issues/detail?id=3431.
78 return __libc_stack_end;
79 #else
80 ASSERT_NOT_REACHED();
81 return 0;
82 #endif
83 #elif OS(MACOSX)
84 return pthread_get_stackaddr_np(pthread_self());
85 #elif OS(WIN) && COMPILER(MSVC)
86 // On Windows stack limits for the current thread are available in
87 // the thread information block (TIB). Its fields can be accessed through
88 // FS segment register on x86 and GS segment register on x86_64.
89 #ifdef _WIN64
90 return reinterpret_cast<void*>(__readgsqword(offsetof(NT_TIB64, StackBase)));
91 #else
92 return reinterpret_cast<void*>(__readfsdword(offsetof(NT_TIB, StackBase)));
93 #endif
94 #else
95 #error Unsupported getStackStart on this platform.
96 #endif
97 }
98
99 // The maximum number of WrapperPersistentRegions to keep around in the
100 // m_pooledWrapperPersistentRegions pool.
101 static const size_t MaxPooledWrapperPersistentRegionCount = 2;
102
103 WTF::ThreadSpecific<ThreadState*>* ThreadState::s_threadSpecific = 0;
104 uint8_t ThreadState::s_mainThreadStateStorage[sizeof(ThreadState)];
105 SafePointBarrier* ThreadState::s_safePointBarrier = 0;
106 bool ThreadState::s_inGC = false;
107
threadAttachMutex()108 static Mutex& threadAttachMutex()
109 {
110 AtomicallyInitializedStatic(Mutex&, mutex = *new Mutex);
111 return mutex;
112 }
113
lockingTimeout()114 static double lockingTimeout()
115 {
116 // Wait time for parking all threads is at most 100 MS.
117 return 0.100;
118 }
119
120
121 typedef void (*PushAllRegistersCallback)(SafePointBarrier*, ThreadState*, intptr_t*);
122 extern "C" void pushAllRegisters(SafePointBarrier*, ThreadState*, PushAllRegistersCallback);
123
124 class SafePointBarrier {
125 public:
SafePointBarrier()126 SafePointBarrier() : m_canResume(1), m_unparkedThreadCount(0) { }
~SafePointBarrier()127 ~SafePointBarrier() { }
128
129 // Request other attached threads that are not at safe points to park themselves on safepoints.
parkOthers()130 bool parkOthers()
131 {
132 ASSERT(ThreadState::current()->isAtSafePoint());
133
134 // Lock threadAttachMutex() to prevent threads from attaching.
135 threadAttachMutex().lock();
136
137 ThreadState::AttachedThreadStateSet& threads = ThreadState::attachedThreads();
138
139 MutexLocker locker(m_mutex);
140 atomicAdd(&m_unparkedThreadCount, threads.size());
141 releaseStore(&m_canResume, 0);
142
143 ThreadState* current = ThreadState::current();
144 for (ThreadState::AttachedThreadStateSet::iterator it = threads.begin(), end = threads.end(); it != end; ++it) {
145 if (*it == current)
146 continue;
147
148 const Vector<ThreadState::Interruptor*>& interruptors = (*it)->interruptors();
149 for (size_t i = 0; i < interruptors.size(); i++)
150 interruptors[i]->requestInterrupt();
151 }
152
153 while (acquireLoad(&m_unparkedThreadCount) > 0) {
154 double expirationTime = currentTime() + lockingTimeout();
155 if (!m_parked.timedWait(m_mutex, expirationTime)) {
156 // One of the other threads did not return to a safepoint within the maximum
157 // time we allow for threads to be parked. Abandon the GC and resume the
158 // currently parked threads.
159 resumeOthers(true);
160 return false;
161 }
162 }
163 return true;
164 }
165
resumeOthers(bool barrierLocked=false)166 void resumeOthers(bool barrierLocked = false)
167 {
168 ThreadState::AttachedThreadStateSet& threads = ThreadState::attachedThreads();
169 atomicSubtract(&m_unparkedThreadCount, threads.size());
170 releaseStore(&m_canResume, 1);
171
172 // FIXME: Resumed threads will all contend for m_mutex just to unlock it
173 // later which is a waste of resources.
174 if (UNLIKELY(barrierLocked)) {
175 m_resume.broadcast();
176 } else {
177 // FIXME: Resumed threads will all contend for
178 // m_mutex just to unlock it later which is a waste of
179 // resources.
180 MutexLocker locker(m_mutex);
181 m_resume.broadcast();
182 }
183
184 ThreadState* current = ThreadState::current();
185 for (ThreadState::AttachedThreadStateSet::iterator it = threads.begin(), end = threads.end(); it != end; ++it) {
186 if (*it == current)
187 continue;
188
189 const Vector<ThreadState::Interruptor*>& interruptors = (*it)->interruptors();
190 for (size_t i = 0; i < interruptors.size(); i++)
191 interruptors[i]->clearInterrupt();
192 }
193
194 threadAttachMutex().unlock();
195 ASSERT(ThreadState::current()->isAtSafePoint());
196 }
197
checkAndPark(ThreadState * state,SafePointAwareMutexLocker * locker=0)198 void checkAndPark(ThreadState* state, SafePointAwareMutexLocker* locker = 0)
199 {
200 ASSERT(!state->isSweepInProgress());
201 if (!acquireLoad(&m_canResume)) {
202 // If we are leaving the safepoint from a SafePointAwareMutexLocker
203 // call out to release the lock before going to sleep. This enables the
204 // lock to be acquired in the sweep phase, e.g. during weak processing
205 // or finalization. The SafePointAwareLocker will reenter the safepoint
206 // and reacquire the lock after leaving this safepoint.
207 if (locker)
208 locker->reset();
209 pushAllRegisters(this, state, parkAfterPushRegisters);
210 }
211 }
212
enterSafePoint(ThreadState * state)213 void enterSafePoint(ThreadState* state)
214 {
215 ASSERT(!state->isSweepInProgress());
216 pushAllRegisters(this, state, enterSafePointAfterPushRegisters);
217 }
218
leaveSafePoint(ThreadState * state,SafePointAwareMutexLocker * locker=0)219 void leaveSafePoint(ThreadState* state, SafePointAwareMutexLocker* locker = 0)
220 {
221 if (atomicIncrement(&m_unparkedThreadCount) > 0)
222 checkAndPark(state, locker);
223 }
224
225 private:
doPark(ThreadState * state,intptr_t * stackEnd)226 void doPark(ThreadState* state, intptr_t* stackEnd)
227 {
228 state->recordStackEnd(stackEnd);
229 MutexLocker locker(m_mutex);
230 if (!atomicDecrement(&m_unparkedThreadCount))
231 m_parked.signal();
232 while (!acquireLoad(&m_canResume))
233 m_resume.wait(m_mutex);
234 atomicIncrement(&m_unparkedThreadCount);
235 }
236
parkAfterPushRegisters(SafePointBarrier * barrier,ThreadState * state,intptr_t * stackEnd)237 static void parkAfterPushRegisters(SafePointBarrier* barrier, ThreadState* state, intptr_t* stackEnd)
238 {
239 barrier->doPark(state, stackEnd);
240 }
241
doEnterSafePoint(ThreadState * state,intptr_t * stackEnd)242 void doEnterSafePoint(ThreadState* state, intptr_t* stackEnd)
243 {
244 state->recordStackEnd(stackEnd);
245 state->copyStackUntilSafePointScope();
246 // m_unparkedThreadCount tracks amount of unparked threads. It is
247 // positive if and only if we have requested other threads to park
248 // at safe-points in preparation for GC. The last thread to park
249 // itself will make the counter hit zero and should notify GC thread
250 // that it is safe to proceed.
251 // If no other thread is waiting for other threads to park then
252 // this counter can be negative: if N threads are at safe-points
253 // the counter will be -N.
254 if (!atomicDecrement(&m_unparkedThreadCount)) {
255 MutexLocker locker(m_mutex);
256 m_parked.signal(); // Safe point reached.
257 }
258 }
259
enterSafePointAfterPushRegisters(SafePointBarrier * barrier,ThreadState * state,intptr_t * stackEnd)260 static void enterSafePointAfterPushRegisters(SafePointBarrier* barrier, ThreadState* state, intptr_t* stackEnd)
261 {
262 barrier->doEnterSafePoint(state, stackEnd);
263 }
264
265 volatile int m_canResume;
266 volatile int m_unparkedThreadCount;
267 Mutex m_mutex;
268 ThreadCondition m_parked;
269 ThreadCondition m_resume;
270 };
271
BaseHeapPage(PageMemory * storage,const GCInfo * gcInfo,ThreadState * state)272 BaseHeapPage::BaseHeapPage(PageMemory* storage, const GCInfo* gcInfo, ThreadState* state)
273 : m_storage(storage)
274 , m_gcInfo(gcInfo)
275 , m_threadState(state)
276 , m_terminating(false)
277 , m_tracedAfterOrphaned(false)
278 {
279 ASSERT(isPageHeaderAddress(reinterpret_cast<Address>(this)));
280 }
281
282 // Statically unfold the heap initialization loop so the compiler statically
283 // knows the heap index when using HeapIndexTrait.
284 template<int num> struct InitializeHeaps {
285 static const int index = num - 1;
initblink::InitializeHeaps286 static void init(BaseHeap** heaps, ThreadState* state)
287 {
288 InitializeHeaps<index>::init(heaps, state);
289 heaps[index] = new typename HeapIndexTrait<index>::HeapType(state, index);
290 }
291 };
292 template<> struct InitializeHeaps<0> {
initblink::InitializeHeaps293 static void init(BaseHeap** heaps, ThreadState* state) { }
294 };
295
ThreadState()296 ThreadState::ThreadState()
297 : m_thread(currentThread())
298 , m_liveWrapperPersistents(new WrapperPersistentRegion())
299 , m_pooledWrapperPersistents(0)
300 , m_pooledWrapperPersistentRegionCount(0)
301 , m_persistents(adoptPtr(new PersistentAnchor()))
302 , m_startOfStack(reinterpret_cast<intptr_t*>(getStackStart()))
303 , m_endOfStack(reinterpret_cast<intptr_t*>(getStackStart()))
304 , m_safePointScopeMarker(0)
305 , m_atSafePoint(false)
306 , m_interruptors()
307 , m_gcRequested(false)
308 , m_forcePreciseGCForTesting(false)
309 , m_sweepRequested(0)
310 , m_sweepInProgress(false)
311 , m_noAllocationCount(0)
312 , m_inGC(false)
313 , m_heapContainsCache(adoptPtr(new HeapContainsCache()))
314 , m_isTerminating(false)
315 , m_lowCollectionRate(false)
316 , m_numberOfSweeperTasks(0)
317 #if defined(ADDRESS_SANITIZER)
318 , m_asanFakeStack(__asan_get_current_fake_stack())
319 #endif
320 {
321 ASSERT(!**s_threadSpecific);
322 **s_threadSpecific = this;
323
324 InitializeHeaps<NumberOfHeaps>::init(m_heaps, this);
325
326 m_weakCallbackStack = new CallbackStack();
327
328 if (blink::Platform::current())
329 m_sweeperThread = adoptPtr(blink::Platform::current()->createThread("Blink GC Sweeper"));
330 }
331
~ThreadState()332 ThreadState::~ThreadState()
333 {
334 checkThread();
335 delete m_weakCallbackStack;
336 m_weakCallbackStack = 0;
337 for (int i = 0; i < NumberOfHeaps; i++)
338 delete m_heaps[i];
339 deleteAllValues(m_interruptors);
340 while (m_liveWrapperPersistents) {
341 WrapperPersistentRegion* region = WrapperPersistentRegion::removeHead(&m_liveWrapperPersistents);
342 delete region;
343 }
344 while (m_pooledWrapperPersistents) {
345 WrapperPersistentRegion* region = WrapperPersistentRegion::removeHead(&m_pooledWrapperPersistents);
346 delete region;
347 }
348 **s_threadSpecific = 0;
349 }
350
init()351 void ThreadState::init()
352 {
353 s_threadSpecific = new WTF::ThreadSpecific<ThreadState*>();
354 s_safePointBarrier = new SafePointBarrier;
355 }
356
shutdown()357 void ThreadState::shutdown()
358 {
359 delete s_safePointBarrier;
360 s_safePointBarrier = 0;
361
362 // Thread-local storage shouldn't be disposed, so we don't call ~ThreadSpecific().
363 }
364
attachMainThread()365 void ThreadState::attachMainThread()
366 {
367 RELEASE_ASSERT(!Heap::s_shutdownCalled);
368 MutexLocker locker(threadAttachMutex());
369 ThreadState* state = new(s_mainThreadStateStorage) ThreadState();
370 attachedThreads().add(state);
371 }
372
detachMainThread()373 void ThreadState::detachMainThread()
374 {
375 // Enter a safe point before trying to acquire threadAttachMutex
376 // to avoid dead lock if another thread is preparing for GC, has acquired
377 // threadAttachMutex and waiting for other threads to pause or reach a
378 // safepoint.
379 ThreadState* state = mainThreadState();
380
381 {
382 SafePointAwareMutexLocker locker(threadAttachMutex(), NoHeapPointersOnStack);
383
384 // First add the main thread's heap pages to the orphaned pool.
385 state->cleanupPages();
386
387 // Second detach thread.
388 ASSERT(attachedThreads().contains(state));
389 attachedThreads().remove(state);
390 state->~ThreadState();
391 }
392 shutdownHeapIfNecessary();
393 }
394
shutdownHeapIfNecessary()395 void ThreadState::shutdownHeapIfNecessary()
396 {
397 // We don't need to enter a safe point before acquiring threadAttachMutex
398 // because this thread is already detached.
399
400 MutexLocker locker(threadAttachMutex());
401 // We start shutting down the heap if there is no running thread
402 // and Heap::shutdown() is already called.
403 if (!attachedThreads().size() && Heap::s_shutdownCalled)
404 Heap::doShutdown();
405 }
406
attach()407 void ThreadState::attach()
408 {
409 RELEASE_ASSERT(!Heap::s_shutdownCalled);
410 MutexLocker locker(threadAttachMutex());
411 ThreadState* state = new ThreadState();
412 attachedThreads().add(state);
413 }
414
cleanupPages()415 void ThreadState::cleanupPages()
416 {
417 for (int i = 0; i < NumberOfHeaps; ++i)
418 m_heaps[i]->cleanupPages();
419 }
420
cleanup()421 void ThreadState::cleanup()
422 {
423 for (size_t i = 0; i < m_cleanupTasks.size(); i++)
424 m_cleanupTasks[i]->preCleanup();
425
426 {
427 // Grab the threadAttachMutex to ensure only one thread can shutdown at
428 // a time and that no other thread can do a global GC. It also allows
429 // safe iteration of the attachedThreads set which happens as part of
430 // thread local GC asserts. We enter a safepoint while waiting for the
431 // lock to avoid a dead-lock where another thread has already requested
432 // GC.
433 SafePointAwareMutexLocker locker(threadAttachMutex(), NoHeapPointersOnStack);
434
435 // From here on ignore all conservatively discovered
436 // pointers into the heap owned by this thread.
437 m_isTerminating = true;
438
439 // Set the terminate flag on all heap pages of this thread. This is used to
440 // ensure we don't trace pages on other threads that are not part of the
441 // thread local GC.
442 setupHeapsForTermination();
443
444 // Do thread local GC's as long as the count of thread local Persistents
445 // changes and is above zero.
446 PersistentAnchor* anchor = static_cast<PersistentAnchor*>(m_persistents.get());
447 int oldCount = -1;
448 int currentCount = anchor->numberOfPersistents();
449 ASSERT(currentCount >= 0);
450 while (currentCount != oldCount) {
451 Heap::collectGarbageForTerminatingThread(this);
452 oldCount = currentCount;
453 currentCount = anchor->numberOfPersistents();
454 }
455 // We should not have any persistents left when getting to this point,
456 // if we have it is probably a bug so adding a debug ASSERT to catch this.
457 ASSERT(!currentCount);
458
459 // Add pages to the orphaned page pool to ensure any global GCs from this point
460 // on will not trace objects on this thread's heaps.
461 cleanupPages();
462
463 ASSERT(attachedThreads().contains(this));
464 attachedThreads().remove(this);
465 }
466
467 for (size_t i = 0; i < m_cleanupTasks.size(); i++)
468 m_cleanupTasks[i]->postCleanup();
469 m_cleanupTasks.clear();
470 }
471
472
detach()473 void ThreadState::detach()
474 {
475 ThreadState* state = current();
476 state->cleanup();
477 delete state;
478 shutdownHeapIfNecessary();
479 }
480
visitPersistentRoots(Visitor * visitor)481 void ThreadState::visitPersistentRoots(Visitor* visitor)
482 {
483 TRACE_EVENT0("blink_gc", "ThreadState::visitPersistentRoots");
484 {
485 // All threads are at safepoints so this is not strictly necessary.
486 // However we acquire the mutex to make mutation and traversal of this
487 // list symmetrical.
488 MutexLocker locker(globalRootsMutex());
489 globalRoots()->trace(visitor);
490 }
491
492 AttachedThreadStateSet& threads = attachedThreads();
493 for (AttachedThreadStateSet::iterator it = threads.begin(), end = threads.end(); it != end; ++it)
494 (*it)->visitPersistents(visitor);
495 }
496
visitStackRoots(Visitor * visitor)497 void ThreadState::visitStackRoots(Visitor* visitor)
498 {
499 TRACE_EVENT0("blink_gc", "ThreadState::visitStackRoots");
500 AttachedThreadStateSet& threads = attachedThreads();
501 for (AttachedThreadStateSet::iterator it = threads.begin(), end = threads.end(); it != end; ++it)
502 (*it)->visitStack(visitor);
503 }
504
505 NO_SANITIZE_ADDRESS
visitAsanFakeStackForPointer(Visitor * visitor,Address ptr)506 void ThreadState::visitAsanFakeStackForPointer(Visitor* visitor, Address ptr)
507 {
508 #if defined(ADDRESS_SANITIZER)
509 Address* start = reinterpret_cast<Address*>(m_startOfStack);
510 Address* end = reinterpret_cast<Address*>(m_endOfStack);
511 Address* fakeFrameStart = 0;
512 Address* fakeFrameEnd = 0;
513 Address* maybeFakeFrame = reinterpret_cast<Address*>(ptr);
514 Address* realFrameForFakeFrame =
515 reinterpret_cast<Address*>(
516 __asan_addr_is_in_fake_stack(
517 m_asanFakeStack, maybeFakeFrame,
518 reinterpret_cast<void**>(&fakeFrameStart),
519 reinterpret_cast<void**>(&fakeFrameEnd)));
520 if (realFrameForFakeFrame) {
521 // This is a fake frame from the asan fake stack.
522 if (realFrameForFakeFrame > end && start > realFrameForFakeFrame) {
523 // The real stack address for the asan fake frame is
524 // within the stack range that we need to scan so we need
525 // to visit the values in the fake frame.
526 for (Address* p = fakeFrameStart; p < fakeFrameEnd; p++)
527 Heap::checkAndMarkPointer(visitor, *p);
528 }
529 }
530 #endif
531 }
532
533 NO_SANITIZE_ADDRESS
visitStack(Visitor * visitor)534 void ThreadState::visitStack(Visitor* visitor)
535 {
536 if (m_stackState == NoHeapPointersOnStack)
537 return;
538
539 Address* start = reinterpret_cast<Address*>(m_startOfStack);
540 // If there is a safepoint scope marker we should stop the stack
541 // scanning there to not touch active parts of the stack. Anything
542 // interesting beyond that point is in the safepoint stack copy.
543 // If there is no scope marker the thread is blocked and we should
544 // scan all the way to the recorded end stack pointer.
545 Address* end = reinterpret_cast<Address*>(m_endOfStack);
546 Address* safePointScopeMarker = reinterpret_cast<Address*>(m_safePointScopeMarker);
547 Address* current = safePointScopeMarker ? safePointScopeMarker : end;
548
549 // Ensure that current is aligned by address size otherwise the loop below
550 // will read past start address.
551 current = reinterpret_cast<Address*>(reinterpret_cast<intptr_t>(current) & ~(sizeof(Address) - 1));
552
553 for (; current < start; ++current) {
554 Address ptr = *current;
555 #if defined(MEMORY_SANITIZER)
556 // |ptr| may be uninitialized by design. Mark it as initialized to keep
557 // MSan from complaining.
558 // Note: it may be tempting to get rid of |ptr| and simply use |current|
559 // here, but that would be incorrect. We intentionally use a local
560 // variable because we don't want to unpoison the original stack.
561 __msan_unpoison(&ptr, sizeof(ptr));
562 #endif
563 Heap::checkAndMarkPointer(visitor, ptr);
564 visitAsanFakeStackForPointer(visitor, ptr);
565 }
566
567 for (Vector<Address>::iterator it = m_safePointStackCopy.begin(); it != m_safePointStackCopy.end(); ++it) {
568 Address ptr = *it;
569 #if defined(MEMORY_SANITIZER)
570 // See the comment above.
571 __msan_unpoison(&ptr, sizeof(ptr));
572 #endif
573 Heap::checkAndMarkPointer(visitor, ptr);
574 visitAsanFakeStackForPointer(visitor, ptr);
575 }
576 }
577
visitPersistents(Visitor * visitor)578 void ThreadState::visitPersistents(Visitor* visitor)
579 {
580 m_persistents->trace(visitor);
581 WrapperPersistentRegion::trace(m_liveWrapperPersistents, visitor);
582 }
583
checkAndMarkPointer(Visitor * visitor,Address address)584 bool ThreadState::checkAndMarkPointer(Visitor* visitor, Address address)
585 {
586 // If thread is terminating ignore conservative pointers.
587 if (m_isTerminating)
588 return false;
589
590 // This checks for normal pages and for large objects which span the extent
591 // of several normal pages.
592 BaseHeapPage* page = heapPageFromAddress(address);
593 if (page) {
594 page->checkAndMarkPointer(visitor, address);
595 // Whether or not the pointer was within an object it was certainly
596 // within a page that is part of the heap, so we don't want to ask the
597 // other other heaps or put this address in the
598 // HeapDoesNotContainCache.
599 return true;
600 }
601
602 return false;
603 }
604
605 #if ENABLE(GC_PROFILE_MARKING)
findGCInfo(Address address)606 const GCInfo* ThreadState::findGCInfo(Address address)
607 {
608 BaseHeapPage* page = heapPageFromAddress(address);
609 if (page) {
610 return page->findGCInfo(address);
611 }
612 return 0;
613 }
614 #endif
615
616 #if ENABLE(GC_PROFILE_HEAP)
getClassTag(const GCInfo * gcinfo)617 size_t ThreadState::SnapshotInfo::getClassTag(const GCInfo* gcinfo)
618 {
619 HashMap<const GCInfo*, size_t>::AddResult result = classTags.add(gcinfo, classTags.size());
620 if (result.isNewEntry) {
621 liveCount.append(0);
622 deadCount.append(0);
623 liveSize.append(0);
624 deadSize.append(0);
625 generations.append(Vector<int, 8>());
626 generations.last().fill(0, 8);
627 }
628 return result.storedValue->value;
629 }
630
snapshot()631 void ThreadState::snapshot()
632 {
633 SnapshotInfo info(this);
634 RefPtr<TracedValue> json = TracedValue::create();
635
636 #define SNAPSHOT_HEAP(HeapType) \
637 { \
638 json->beginDictionary(); \
639 json->setString("name", #HeapType); \
640 m_heaps[HeapType##Heap]->snapshot(json.get(), &info); \
641 json->endDictionary(); \
642 json->beginDictionary(); \
643 json->setString("name", #HeapType"NonFinalized"); \
644 m_heaps[HeapType##HeapNonFinalized]->snapshot(json.get(), &info); \
645 json->endDictionary(); \
646 }
647 json->beginArray("heaps");
648 SNAPSHOT_HEAP(General);
649 SNAPSHOT_HEAP(CollectionBacking);
650 FOR_EACH_TYPED_HEAP(SNAPSHOT_HEAP);
651 json->endArray();
652 #undef SNAPSHOT_HEAP
653
654 json->setInteger("allocatedSpace", m_stats.totalAllocatedSpace());
655 json->setInteger("objectSpace", m_stats.totalObjectSpace());
656 json->setInteger("pageCount", info.pageCount);
657 json->setInteger("freeSize", info.freeSize);
658
659 Vector<String> classNameVector(info.classTags.size());
660 for (HashMap<const GCInfo*, size_t>::iterator it = info.classTags.begin(); it != info.classTags.end(); ++it)
661 classNameVector[it->value] = it->key->m_className;
662
663 size_t liveSize = 0;
664 size_t deadSize = 0;
665 json->beginArray("classes");
666 for (size_t i = 0; i < classNameVector.size(); ++i) {
667 json->beginDictionary();
668 json->setString("name", classNameVector[i]);
669 json->setInteger("liveCount", info.liveCount[i]);
670 json->setInteger("deadCount", info.deadCount[i]);
671 json->setInteger("liveSize", info.liveSize[i]);
672 json->setInteger("deadSize", info.deadSize[i]);
673 liveSize += info.liveSize[i];
674 deadSize += info.deadSize[i];
675
676 json->beginArray("generations");
677 for (size_t j = 0; j < heapObjectGenerations; ++j)
678 json->pushInteger(info.generations[i][j]);
679 json->endArray();
680 json->endDictionary();
681 }
682 json->endArray();
683 json->setInteger("liveSize", liveSize);
684 json->setInteger("deadSize", deadSize);
685
686 TRACE_EVENT_OBJECT_SNAPSHOT_WITH_ID("blink_gc", "ThreadState", this, json.release());
687 }
688 #endif
689
pushWeakObjectPointerCallback(void * object,WeakPointerCallback callback)690 void ThreadState::pushWeakObjectPointerCallback(void* object, WeakPointerCallback callback)
691 {
692 CallbackStack::Item* slot = m_weakCallbackStack->allocateEntry();
693 *slot = CallbackStack::Item(object, callback);
694 }
695
popAndInvokeWeakPointerCallback(Visitor * visitor)696 bool ThreadState::popAndInvokeWeakPointerCallback(Visitor* visitor)
697 {
698 // For weak processing we should never reach orphaned pages since orphaned
699 // pages are not traced and thus objects on those pages are never be
700 // registered as objects on orphaned pages. We cannot assert this here since
701 // we might have an off-heap collection. We assert it in
702 // Heap::pushWeakObjectPointerCallback.
703 if (CallbackStack::Item* item = m_weakCallbackStack->pop()) {
704 item->call(visitor);
705 return true;
706 }
707 return false;
708 }
709
takeWrapperPersistentRegion()710 WrapperPersistentRegion* ThreadState::takeWrapperPersistentRegion()
711 {
712 WrapperPersistentRegion* region;
713 if (m_pooledWrapperPersistentRegionCount) {
714 region = WrapperPersistentRegion::removeHead(&m_pooledWrapperPersistents);
715 m_pooledWrapperPersistentRegionCount--;
716 } else {
717 region = new WrapperPersistentRegion();
718 }
719 ASSERT(region);
720 WrapperPersistentRegion::insertHead(&m_liveWrapperPersistents, region);
721 return region;
722 }
723
freeWrapperPersistentRegion(WrapperPersistentRegion * region)724 void ThreadState::freeWrapperPersistentRegion(WrapperPersistentRegion* region)
725 {
726 if (!region->removeIfNotLast(&m_liveWrapperPersistents))
727 return;
728
729 // Region was removed, ie. it was not the last region in the list.
730 if (m_pooledWrapperPersistentRegionCount < MaxPooledWrapperPersistentRegionCount) {
731 WrapperPersistentRegion::insertHead(&m_pooledWrapperPersistents, region);
732 m_pooledWrapperPersistentRegionCount++;
733 } else {
734 delete region;
735 }
736 }
737
globalRoots()738 PersistentNode* ThreadState::globalRoots()
739 {
740 AtomicallyInitializedStatic(PersistentNode*, anchor = new PersistentAnchor);
741 return anchor;
742 }
743
globalRootsMutex()744 Mutex& ThreadState::globalRootsMutex()
745 {
746 AtomicallyInitializedStatic(Mutex&, mutex = *new Mutex);
747 return mutex;
748 }
749
750 // Trigger garbage collection on a 50% increase in size, but not for
751 // less than 512kbytes.
increasedEnoughToGC(size_t newSize,size_t oldSize)752 bool ThreadState::increasedEnoughToGC(size_t newSize, size_t oldSize)
753 {
754 if (newSize < 1 << 19)
755 return false;
756 size_t limit = oldSize + (oldSize >> 1);
757 return newSize > limit;
758 }
759
760 // FIXME: The heuristics are local for a thread at this
761 // point. Consider using heuristics that take memory for all threads
762 // into account.
shouldGC()763 bool ThreadState::shouldGC()
764 {
765 // Do not GC during sweeping. We allow allocation during
766 // finalization, but those allocations are not allowed
767 // to lead to nested garbage collections.
768 return !m_sweepInProgress && increasedEnoughToGC(m_stats.totalObjectSpace(), m_statsAfterLastGC.totalObjectSpace());
769 }
770
771 // Trigger conservative garbage collection on a 100% increase in size,
772 // but not for less than 4Mbytes. If the system currently has a low
773 // collection rate, then require a 300% increase in size.
increasedEnoughToForceConservativeGC(size_t newSize,size_t oldSize)774 bool ThreadState::increasedEnoughToForceConservativeGC(size_t newSize, size_t oldSize)
775 {
776 if (newSize < 1 << 22)
777 return false;
778 size_t limit = (m_lowCollectionRate ? 4 : 2) * oldSize;
779 return newSize > limit;
780 }
781
782 // FIXME: The heuristics are local for a thread at this
783 // point. Consider using heuristics that take memory for all threads
784 // into account.
shouldForceConservativeGC()785 bool ThreadState::shouldForceConservativeGC()
786 {
787 // Do not GC during sweeping. We allow allocation during
788 // finalization, but those allocations are not allowed
789 // to lead to nested garbage collections.
790 return !m_sweepInProgress && increasedEnoughToForceConservativeGC(m_stats.totalObjectSpace(), m_statsAfterLastGC.totalObjectSpace());
791 }
792
sweepRequested()793 bool ThreadState::sweepRequested()
794 {
795 ASSERT(isAnyThreadInGC() || checkThread());
796 return m_sweepRequested;
797 }
798
setSweepRequested()799 void ThreadState::setSweepRequested()
800 {
801 // Sweep requested is set from the thread that initiates garbage
802 // collection which could be different from the thread for this
803 // thread state. Therefore the setting of m_sweepRequested needs a
804 // barrier.
805 atomicTestAndSetToOne(&m_sweepRequested);
806 }
807
clearSweepRequested()808 void ThreadState::clearSweepRequested()
809 {
810 checkThread();
811 m_sweepRequested = 0;
812 }
813
gcRequested()814 bool ThreadState::gcRequested()
815 {
816 checkThread();
817 return m_gcRequested;
818 }
819
setGCRequested()820 void ThreadState::setGCRequested()
821 {
822 checkThread();
823 m_gcRequested = true;
824 }
825
clearGCRequested()826 void ThreadState::clearGCRequested()
827 {
828 checkThread();
829 m_gcRequested = false;
830 }
831
performPendingGC(StackState stackState)832 void ThreadState::performPendingGC(StackState stackState)
833 {
834 if (stackState == NoHeapPointersOnStack) {
835 if (forcePreciseGCForTesting()) {
836 setForcePreciseGCForTesting(false);
837 Heap::collectAllGarbage();
838 } else if (gcRequested()) {
839 Heap::collectGarbage(NoHeapPointersOnStack);
840 }
841 }
842 }
843
setForcePreciseGCForTesting(bool value)844 void ThreadState::setForcePreciseGCForTesting(bool value)
845 {
846 checkThread();
847 m_forcePreciseGCForTesting = value;
848 }
849
forcePreciseGCForTesting()850 bool ThreadState::forcePreciseGCForTesting()
851 {
852 checkThread();
853 return m_forcePreciseGCForTesting;
854 }
855
makeConsistentForSweeping()856 void ThreadState::makeConsistentForSweeping()
857 {
858 for (int i = 0; i < NumberOfHeaps; i++)
859 m_heaps[i]->makeConsistentForSweeping();
860 }
861
862 #if ENABLE(ASSERT)
isConsistentForSweeping()863 bool ThreadState::isConsistentForSweeping()
864 {
865 for (int i = 0; i < NumberOfHeaps; i++) {
866 if (!m_heaps[i]->isConsistentForSweeping())
867 return false;
868 }
869 return true;
870 }
871 #endif
872
prepareForGC()873 void ThreadState::prepareForGC()
874 {
875 for (int i = 0; i < NumberOfHeaps; i++) {
876 BaseHeap* heap = m_heaps[i];
877 heap->makeConsistentForSweeping();
878 // If a new GC is requested before this thread got around to sweep, ie. due to the
879 // thread doing a long running operation, we clear the mark bits and mark any of
880 // the dead objects as dead. The latter is used to ensure the next GC marking does
881 // not trace already dead objects. If we trace a dead object we could end up tracing
882 // into garbage or the middle of another object via the newly conservatively found
883 // object.
884 if (sweepRequested())
885 heap->clearLiveAndMarkDead();
886 }
887 setSweepRequested();
888 }
889
setupHeapsForTermination()890 void ThreadState::setupHeapsForTermination()
891 {
892 for (int i = 0; i < NumberOfHeaps; i++)
893 m_heaps[i]->prepareHeapForTermination();
894 }
895
heapPageFromAddress(Address address)896 BaseHeapPage* ThreadState::heapPageFromAddress(Address address)
897 {
898 BaseHeapPage* cachedPage = heapContainsCache()->lookup(address);
899 #if !ENABLE(ASSERT)
900 if (cachedPage)
901 return cachedPage;
902 #endif
903
904 for (int i = 0; i < NumberOfHeaps; i++) {
905 BaseHeapPage* page = m_heaps[i]->heapPageFromAddress(address);
906 if (page) {
907 // Asserts that make sure heapPageFromAddress takes addresses from
908 // the whole aligned blinkPageSize memory area. This is necessary
909 // for the negative cache to work.
910 ASSERT(page->isLargeObject() || page == m_heaps[i]->heapPageFromAddress(roundToBlinkPageStart(address)));
911 if (roundToBlinkPageStart(address) != roundToBlinkPageEnd(address))
912 ASSERT(page->isLargeObject() || page == m_heaps[i]->heapPageFromAddress(roundToBlinkPageEnd(address) - 1));
913 ASSERT(!cachedPage || page == cachedPage);
914 if (!cachedPage)
915 heapContainsCache()->addEntry(address, page);
916 return page;
917 }
918 }
919 ASSERT(!cachedPage);
920 return 0;
921 }
922
getStats(HeapStats & stats)923 void ThreadState::getStats(HeapStats& stats)
924 {
925 stats = m_stats;
926 #if ENABLE(ASSERT)
927 if (isConsistentForSweeping()) {
928 HeapStats scannedStats;
929 for (int i = 0; i < NumberOfHeaps; i++)
930 m_heaps[i]->getScannedStats(scannedStats);
931 ASSERT(scannedStats == stats);
932 }
933 #endif
934 }
935
stopThreads()936 bool ThreadState::stopThreads()
937 {
938 return s_safePointBarrier->parkOthers();
939 }
940
resumeThreads()941 void ThreadState::resumeThreads()
942 {
943 s_safePointBarrier->resumeOthers();
944 }
945
safePoint(StackState stackState)946 void ThreadState::safePoint(StackState stackState)
947 {
948 checkThread();
949 performPendingGC(stackState);
950 ASSERT(!m_atSafePoint);
951 m_stackState = stackState;
952 m_atSafePoint = true;
953 s_safePointBarrier->checkAndPark(this);
954 m_atSafePoint = false;
955 m_stackState = HeapPointersOnStack;
956 performPendingSweep();
957 }
958
959 #ifdef ADDRESS_SANITIZER
960 // When we are running under AddressSanitizer with detect_stack_use_after_return=1
961 // then stack marker obtained from SafePointScope will point into a fake stack.
962 // Detect this case by checking if it falls in between current stack frame
963 // and stack start and use an arbitrary high enough value for it.
964 // Don't adjust stack marker in any other case to match behavior of code running
965 // without AddressSanitizer.
adjustScopeMarkerForAdressSanitizer(void * scopeMarker)966 NO_SANITIZE_ADDRESS static void* adjustScopeMarkerForAdressSanitizer(void* scopeMarker)
967 {
968 Address start = reinterpret_cast<Address>(getStackStart());
969 Address end = reinterpret_cast<Address>(&start);
970 RELEASE_ASSERT(end < start);
971
972 if (end <= scopeMarker && scopeMarker < start)
973 return scopeMarker;
974
975 // 256 is as good an approximation as any else.
976 const size_t bytesToCopy = sizeof(Address) * 256;
977 if (static_cast<size_t>(start - end) < bytesToCopy)
978 return start;
979
980 return end + bytesToCopy;
981 }
982 #endif
983
enterSafePoint(StackState stackState,void * scopeMarker)984 void ThreadState::enterSafePoint(StackState stackState, void* scopeMarker)
985 {
986 #ifdef ADDRESS_SANITIZER
987 if (stackState == HeapPointersOnStack)
988 scopeMarker = adjustScopeMarkerForAdressSanitizer(scopeMarker);
989 #endif
990 ASSERT(stackState == NoHeapPointersOnStack || scopeMarker);
991 performPendingGC(stackState);
992 checkThread();
993 ASSERT(!m_atSafePoint);
994 m_atSafePoint = true;
995 m_stackState = stackState;
996 m_safePointScopeMarker = scopeMarker;
997 s_safePointBarrier->enterSafePoint(this);
998 }
999
leaveSafePoint(SafePointAwareMutexLocker * locker)1000 void ThreadState::leaveSafePoint(SafePointAwareMutexLocker* locker)
1001 {
1002 checkThread();
1003 ASSERT(m_atSafePoint);
1004 s_safePointBarrier->leaveSafePoint(this, locker);
1005 m_atSafePoint = false;
1006 m_stackState = HeapPointersOnStack;
1007 clearSafePointScopeMarker();
1008 performPendingSweep();
1009 }
1010
copyStackUntilSafePointScope()1011 void ThreadState::copyStackUntilSafePointScope()
1012 {
1013 if (!m_safePointScopeMarker || m_stackState == NoHeapPointersOnStack)
1014 return;
1015
1016 Address* to = reinterpret_cast<Address*>(m_safePointScopeMarker);
1017 Address* from = reinterpret_cast<Address*>(m_endOfStack);
1018 RELEASE_ASSERT(from < to);
1019 RELEASE_ASSERT(to <= reinterpret_cast<Address*>(m_startOfStack));
1020 size_t slotCount = static_cast<size_t>(to - from);
1021 // Catch potential performance issues.
1022 #if defined(LEAK_SANITIZER) || defined(ADDRESS_SANITIZER)
1023 // ASan/LSan use more space on the stack and we therefore
1024 // increase the allowed stack copying for those builds.
1025 ASSERT(slotCount < 2048);
1026 #else
1027 ASSERT(slotCount < 1024);
1028 #endif
1029
1030 ASSERT(!m_safePointStackCopy.size());
1031 m_safePointStackCopy.resize(slotCount);
1032 for (size_t i = 0; i < slotCount; ++i) {
1033 m_safePointStackCopy[i] = from[i];
1034 }
1035 }
1036
registerSweepingTask()1037 void ThreadState::registerSweepingTask()
1038 {
1039 MutexLocker locker(m_sweepMutex);
1040 ++m_numberOfSweeperTasks;
1041 }
1042
unregisterSweepingTask()1043 void ThreadState::unregisterSweepingTask()
1044 {
1045 MutexLocker locker(m_sweepMutex);
1046 ASSERT(m_numberOfSweeperTasks > 0);
1047 if (!--m_numberOfSweeperTasks)
1048 m_sweepThreadCondition.signal();
1049 }
1050
waitUntilSweepersDone()1051 void ThreadState::waitUntilSweepersDone()
1052 {
1053 MutexLocker locker(m_sweepMutex);
1054 while (m_numberOfSweeperTasks > 0)
1055 m_sweepThreadCondition.wait(m_sweepMutex);
1056 }
1057
1058
1059 class SweepNonFinalizedHeapTask FINAL : public WebThread::Task {
1060 public:
SweepNonFinalizedHeapTask(ThreadState * state,BaseHeap * heap,HeapStats * stats)1061 SweepNonFinalizedHeapTask(ThreadState* state, BaseHeap* heap, HeapStats* stats)
1062 : m_threadState(state)
1063 , m_heap(heap)
1064 , m_stats(stats)
1065 {
1066 m_threadState->registerSweepingTask();
1067 }
1068
~SweepNonFinalizedHeapTask()1069 virtual ~SweepNonFinalizedHeapTask()
1070 {
1071 m_threadState->unregisterSweepingTask();
1072 }
1073
run()1074 virtual void run() { m_heap->sweep(m_stats); }
1075
1076 private:
1077 ThreadState* m_threadState;
1078 BaseHeap* m_heap;
1079 HeapStats* m_stats;
1080 };
1081
performPendingSweep()1082 void ThreadState::performPendingSweep()
1083 {
1084 if (!sweepRequested())
1085 return;
1086
1087 #if ENABLE(GC_PROFILE_HEAP)
1088 // We snapshot the heap prior to sweeping to get numbers for both resources
1089 // that have been allocated since the last GC and for resources that are
1090 // going to be freed.
1091 bool gcTracingEnabled;
1092 TRACE_EVENT_CATEGORY_GROUP_ENABLED("blink_gc", &gcTracingEnabled);
1093 if (gcTracingEnabled && m_stats.totalObjectSpace() > 0)
1094 snapshot();
1095 #endif
1096
1097 TRACE_EVENT0("blink_gc", "ThreadState::performPendingSweep");
1098
1099 double timeStamp = WTF::currentTimeMS();
1100 const char* samplingState = TRACE_EVENT_GET_SAMPLING_STATE();
1101 if (isMainThread()) {
1102 ScriptForbiddenScope::enter();
1103 TRACE_EVENT_SET_SAMPLING_STATE("blink", "BlinkGCSweeping");
1104 }
1105
1106 size_t objectSpaceBeforeSweep = m_stats.totalObjectSpace();
1107 {
1108 NoSweepScope scope(this);
1109
1110 // Disallow allocation during weak processing.
1111 enterNoAllocationScope();
1112 {
1113 TRACE_EVENT0("blink_gc", "ThreadState::threadLocalWeakProcessing");
1114 // Perform thread-specific weak processing.
1115 while (popAndInvokeWeakPointerCallback(Heap::s_markingVisitor)) { }
1116 }
1117 leaveNoAllocationScope();
1118
1119 // Perform sweeping and finalization.
1120
1121 // Sweeping will recalculate the stats
1122 m_stats.clear();
1123
1124 // Sweep the non-finalized heap pages on multiple threads.
1125 // Attempt to load-balance by having the sweeper thread sweep as
1126 // close to half of the pages as possible.
1127 int nonFinalizedPages = 0;
1128 for (int i = 0; i < NumberOfNonFinalizedHeaps; i++)
1129 nonFinalizedPages += m_heaps[FirstNonFinalizedHeap + i]->normalPageCount();
1130
1131 int finalizedPages = 0;
1132 for (int i = 0; i < NumberOfFinalizedHeaps; i++)
1133 finalizedPages += m_heaps[FirstFinalizedHeap + i]->normalPageCount();
1134
1135 int pagesToSweepInParallel = nonFinalizedPages < finalizedPages ? nonFinalizedPages : ((nonFinalizedPages + finalizedPages) / 2);
1136
1137 // Start the sweeper thread for the non finalized heaps. No
1138 // finalizers need to run and therefore the pages can be
1139 // swept on other threads.
1140 static const int minNumberOfPagesForParallelSweep = 10;
1141 HeapStats heapStatsVector[NumberOfNonFinalizedHeaps];
1142 BaseHeap* splitOffHeaps[NumberOfNonFinalizedHeaps] = { 0 };
1143 for (int i = 0; i < NumberOfNonFinalizedHeaps && pagesToSweepInParallel > 0; i++) {
1144 BaseHeap* heap = m_heaps[FirstNonFinalizedHeap + i];
1145 int pageCount = heap->normalPageCount();
1146 // Only use the sweeper thread if it exists and there are
1147 // pages to sweep.
1148 if (m_sweeperThread && pageCount > minNumberOfPagesForParallelSweep) {
1149 // Create a new thread heap instance to make sure that the
1150 // state modified while sweeping is separate for the
1151 // sweeper thread and the owner thread.
1152 int pagesToSplitOff = std::min(pageCount, pagesToSweepInParallel);
1153 pagesToSweepInParallel -= pagesToSplitOff;
1154 BaseHeap* splitOff = heap->split(pagesToSplitOff);
1155 splitOffHeaps[i] = splitOff;
1156 HeapStats* stats = &heapStatsVector[i];
1157 m_sweeperThread->postTask(new SweepNonFinalizedHeapTask(this, splitOff, stats));
1158 }
1159 }
1160
1161 {
1162 // Sweep the remainder of the non-finalized pages (or all of them
1163 // if there is no sweeper thread).
1164 TRACE_EVENT0("blink_gc", "ThreadState::sweepNonFinalizedHeaps");
1165 for (int i = 0; i < NumberOfNonFinalizedHeaps; i++) {
1166 HeapStats stats;
1167 m_heaps[FirstNonFinalizedHeap + i]->sweep(&stats);
1168 m_stats.add(&stats);
1169 }
1170 }
1171
1172 {
1173 // Sweep the finalized pages.
1174 TRACE_EVENT0("blink_gc", "ThreadState::sweepFinalizedHeaps");
1175 for (int i = 0; i < NumberOfFinalizedHeaps; i++) {
1176 HeapStats stats;
1177 m_heaps[FirstFinalizedHeap + i]->sweep(&stats);
1178 m_stats.add(&stats);
1179 }
1180 }
1181
1182 // Wait for the sweeper threads and update the heap stats with the
1183 // stats for the heap portions swept by those threads.
1184 waitUntilSweepersDone();
1185 for (int i = 0; i < NumberOfNonFinalizedHeaps; i++) {
1186 m_stats.add(&heapStatsVector[i]);
1187 if (BaseHeap* splitOff = splitOffHeaps[i])
1188 m_heaps[FirstNonFinalizedHeap + i]->merge(splitOff);
1189 }
1190
1191 for (int i = 0; i < NumberOfHeaps; i++)
1192 m_heaps[i]->postSweepProcessing();
1193
1194 getStats(m_statsAfterLastGC);
1195
1196 } // End NoSweepScope
1197 clearGCRequested();
1198 clearSweepRequested();
1199 // If we collected less than 50% of objects, record that the
1200 // collection rate is low which we use to determine when to
1201 // perform the next GC.
1202 setLowCollectionRate(m_stats.totalObjectSpace() > (objectSpaceBeforeSweep >> 1));
1203
1204 if (blink::Platform::current()) {
1205 blink::Platform::current()->histogramCustomCounts("BlinkGC.PerformPendingSweep", WTF::currentTimeMS() - timeStamp, 0, 10 * 1000, 50);
1206 }
1207
1208 if (isMainThread()) {
1209 TRACE_EVENT_SET_NONCONST_SAMPLING_STATE(samplingState);
1210 ScriptForbiddenScope::exit();
1211 }
1212 }
1213
addInterruptor(Interruptor * interruptor)1214 void ThreadState::addInterruptor(Interruptor* interruptor)
1215 {
1216 SafePointScope scope(HeapPointersOnStack, SafePointScope::AllowNesting);
1217
1218 {
1219 MutexLocker locker(threadAttachMutex());
1220 m_interruptors.append(interruptor);
1221 }
1222 }
1223
removeInterruptor(Interruptor * interruptor)1224 void ThreadState::removeInterruptor(Interruptor* interruptor)
1225 {
1226 SafePointScope scope(HeapPointersOnStack, SafePointScope::AllowNesting);
1227
1228 {
1229 MutexLocker locker(threadAttachMutex());
1230 size_t index = m_interruptors.find(interruptor);
1231 RELEASE_ASSERT(index >= 0);
1232 m_interruptors.remove(index);
1233 }
1234 }
1235
onInterrupted()1236 void ThreadState::Interruptor::onInterrupted()
1237 {
1238 ThreadState* state = ThreadState::current();
1239 ASSERT(state);
1240 ASSERT(!state->isAtSafePoint());
1241 state->safePoint(HeapPointersOnStack);
1242 }
1243
attachedThreads()1244 ThreadState::AttachedThreadStateSet& ThreadState::attachedThreads()
1245 {
1246 DEFINE_STATIC_LOCAL(AttachedThreadStateSet, threads, ());
1247 return threads;
1248 }
1249
1250 #if ENABLE(GC_PROFILE_MARKING)
findGCInfoFromAllThreads(Address address)1251 const GCInfo* ThreadState::findGCInfoFromAllThreads(Address address)
1252 {
1253 bool needLockForIteration = !isAnyThreadInGC();
1254 if (needLockForIteration)
1255 threadAttachMutex().lock();
1256
1257 ThreadState::AttachedThreadStateSet& threads = attachedThreads();
1258 for (ThreadState::AttachedThreadStateSet::iterator it = threads.begin(), end = threads.end(); it != end; ++it) {
1259 if (const GCInfo* gcInfo = (*it)->findGCInfo(address)) {
1260 if (needLockForIteration)
1261 threadAttachMutex().unlock();
1262 return gcInfo;
1263 }
1264 }
1265 if (needLockForIteration)
1266 threadAttachMutex().unlock();
1267 return 0;
1268 }
1269 #endif
1270
1271 }
1272