1 // Copyright 2015 the V8 project authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #ifndef V8_HEAP_SCAVENGER_INL_H_
6 #define V8_HEAP_SCAVENGER_INL_H_
7
8 #include "src/heap/scavenger.h"
9
10 #include "src/heap/local-allocator-inl.h"
11 #include "src/objects-inl.h"
12 #include "src/objects/map.h"
13
14 namespace v8 {
15 namespace internal {
16
17 // White list for objects that for sure only contain data.
ContainsOnlyData(VisitorId visitor_id)18 bool Scavenger::ContainsOnlyData(VisitorId visitor_id) {
19 switch (visitor_id) {
20 case kVisitSeqOneByteString:
21 return true;
22 case kVisitSeqTwoByteString:
23 return true;
24 case kVisitByteArray:
25 return true;
26 case kVisitFixedDoubleArray:
27 return true;
28 case kVisitDataObject:
29 return true;
30 default:
31 break;
32 }
33 return false;
34 }
35
PageMemoryFence(MaybeObject * object)36 void Scavenger::PageMemoryFence(MaybeObject* object) {
37 #ifdef THREAD_SANITIZER
38 // Perform a dummy acquire load to tell TSAN that there is no data race
39 // with page initialization.
40 HeapObject* heap_object;
41 if (object->ToStrongOrWeakHeapObject(&heap_object)) {
42 MemoryChunk* chunk = MemoryChunk::FromAddress(heap_object->address());
43 CHECK_NOT_NULL(chunk->synchronized_heap());
44 }
45 #endif
46 }
47
MigrateObject(Map * map,HeapObject * source,HeapObject * target,int size)48 bool Scavenger::MigrateObject(Map* map, HeapObject* source, HeapObject* target,
49 int size) {
50 // Copy the content of source to target.
51 target->set_map_word(MapWord::FromMap(map));
52 heap()->CopyBlock(target->address() + kPointerSize,
53 source->address() + kPointerSize, size - kPointerSize);
54
55 HeapObject* old = base::AsAtomicPointer::Release_CompareAndSwap(
56 reinterpret_cast<HeapObject**>(source->address()), map,
57 MapWord::FromForwardingAddress(target).ToMap());
58 if (old != map) {
59 // Other task migrated the object.
60 return false;
61 }
62
63 if (V8_UNLIKELY(is_logging_)) {
64 heap()->OnMoveEvent(target, source, size);
65 }
66
67 if (is_incremental_marking_) {
68 heap()->incremental_marking()->TransferColor(source, target);
69 }
70 heap()->UpdateAllocationSite(map, source, &local_pretenuring_feedback_);
71 return true;
72 }
73
SemiSpaceCopyObject(Map * map,HeapObjectReference ** slot,HeapObject * object,int object_size)74 bool Scavenger::SemiSpaceCopyObject(Map* map, HeapObjectReference** slot,
75 HeapObject* object, int object_size) {
76 DCHECK(heap()->AllowedToBeMigrated(object, NEW_SPACE));
77 AllocationAlignment alignment = HeapObject::RequiredAlignment(map);
78 AllocationResult allocation =
79 allocator_.Allocate(NEW_SPACE, object_size, alignment);
80
81 HeapObject* target = nullptr;
82 if (allocation.To(&target)) {
83 DCHECK(heap()->incremental_marking()->non_atomic_marking_state()->IsWhite(
84 target));
85 const bool self_success = MigrateObject(map, object, target, object_size);
86 if (!self_success) {
87 allocator_.FreeLast(NEW_SPACE, target, object_size);
88 MapWord map_word = object->map_word();
89 HeapObjectReference::Update(slot, map_word.ToForwardingAddress());
90 return true;
91 }
92 HeapObjectReference::Update(slot, target);
93
94 copied_list_.Push(ObjectAndSize(target, object_size));
95 copied_size_ += object_size;
96 return true;
97 }
98 return false;
99 }
100
PromoteObject(Map * map,HeapObjectReference ** slot,HeapObject * object,int object_size)101 bool Scavenger::PromoteObject(Map* map, HeapObjectReference** slot,
102 HeapObject* object, int object_size) {
103 AllocationAlignment alignment = HeapObject::RequiredAlignment(map);
104 AllocationResult allocation =
105 allocator_.Allocate(OLD_SPACE, object_size, alignment);
106
107 HeapObject* target = nullptr;
108 if (allocation.To(&target)) {
109 DCHECK(heap()->incremental_marking()->non_atomic_marking_state()->IsWhite(
110 target));
111 const bool self_success = MigrateObject(map, object, target, object_size);
112 if (!self_success) {
113 allocator_.FreeLast(OLD_SPACE, target, object_size);
114 MapWord map_word = object->map_word();
115 HeapObjectReference::Update(slot, map_word.ToForwardingAddress());
116 return true;
117 }
118 HeapObjectReference::Update(slot, target);
119 if (!ContainsOnlyData(map->visitor_id())) {
120 promotion_list_.Push(ObjectAndSize(target, object_size));
121 }
122 promoted_size_ += object_size;
123 return true;
124 }
125 return false;
126 }
127
EvacuateObjectDefault(Map * map,HeapObjectReference ** slot,HeapObject * object,int object_size)128 void Scavenger::EvacuateObjectDefault(Map* map, HeapObjectReference** slot,
129 HeapObject* object, int object_size) {
130 SLOW_DCHECK(object_size <= Page::kAllocatableMemory);
131 SLOW_DCHECK(object->SizeFromMap(map) == object_size);
132
133 if (!heap()->ShouldBePromoted(object->address())) {
134 // A semi-space copy may fail due to fragmentation. In that case, we
135 // try to promote the object.
136 if (SemiSpaceCopyObject(map, slot, object, object_size)) return;
137 }
138
139 if (PromoteObject(map, slot, object, object_size)) return;
140
141 // If promotion failed, we try to copy the object to the other semi-space
142 if (SemiSpaceCopyObject(map, slot, object, object_size)) return;
143
144 heap()->FatalProcessOutOfMemory("Scavenger: semi-space copy");
145 }
146
EvacuateThinString(Map * map,HeapObject ** slot,ThinString * object,int object_size)147 void Scavenger::EvacuateThinString(Map* map, HeapObject** slot,
148 ThinString* object, int object_size) {
149 if (!is_incremental_marking_) {
150 // Loading actual is fine in a parallel setting is there is no write.
151 String* actual = object->actual();
152 object->set_length(0);
153 *slot = actual;
154 // ThinStrings always refer to internalized strings, which are
155 // always in old space.
156 DCHECK(!Heap::InNewSpace(actual));
157 base::AsAtomicPointer::Relaxed_Store(
158 reinterpret_cast<Map**>(object->address()),
159 MapWord::FromForwardingAddress(actual).ToMap());
160 return;
161 }
162
163 EvacuateObjectDefault(map, reinterpret_cast<HeapObjectReference**>(slot),
164 object, object_size);
165 }
166
EvacuateShortcutCandidate(Map * map,HeapObject ** slot,ConsString * object,int object_size)167 void Scavenger::EvacuateShortcutCandidate(Map* map, HeapObject** slot,
168 ConsString* object, int object_size) {
169 DCHECK(IsShortcutCandidate(map->instance_type()));
170 if (!is_incremental_marking_ &&
171 object->unchecked_second() == ReadOnlyRoots(heap()).empty_string()) {
172 HeapObject* first = HeapObject::cast(object->unchecked_first());
173
174 *slot = first;
175
176 if (!Heap::InNewSpace(first)) {
177 base::AsAtomicPointer::Relaxed_Store(
178 reinterpret_cast<Map**>(object->address()),
179 MapWord::FromForwardingAddress(first).ToMap());
180 return;
181 }
182
183 MapWord first_word = first->map_word();
184 if (first_word.IsForwardingAddress()) {
185 HeapObject* target = first_word.ToForwardingAddress();
186
187 *slot = target;
188 base::AsAtomicPointer::Relaxed_Store(
189 reinterpret_cast<Map**>(object->address()),
190 MapWord::FromForwardingAddress(target).ToMap());
191 return;
192 }
193 Map* map = first_word.ToMap();
194 EvacuateObjectDefault(map, reinterpret_cast<HeapObjectReference**>(slot),
195 first, first->SizeFromMap(map));
196 base::AsAtomicPointer::Relaxed_Store(
197 reinterpret_cast<Map**>(object->address()),
198 MapWord::FromForwardingAddress(*slot).ToMap());
199 return;
200 }
201
202 EvacuateObjectDefault(map, reinterpret_cast<HeapObjectReference**>(slot),
203 object, object_size);
204 }
205
EvacuateObject(HeapObjectReference ** slot,Map * map,HeapObject * source)206 void Scavenger::EvacuateObject(HeapObjectReference** slot, Map* map,
207 HeapObject* source) {
208 SLOW_DCHECK(Heap::InFromSpace(source));
209 SLOW_DCHECK(!MapWord::FromMap(map).IsForwardingAddress());
210 int size = source->SizeFromMap(map);
211 // Cannot use ::cast() below because that would add checks in debug mode
212 // that require re-reading the map.
213 switch (map->visitor_id()) {
214 case kVisitThinString:
215 // At the moment we don't allow weak pointers to thin strings.
216 DCHECK(!(*slot)->IsWeakHeapObject());
217 EvacuateThinString(map, reinterpret_cast<HeapObject**>(slot),
218 reinterpret_cast<ThinString*>(source), size);
219 break;
220 case kVisitShortcutCandidate:
221 DCHECK(!(*slot)->IsWeakHeapObject());
222 // At the moment we don't allow weak pointers to cons strings.
223 EvacuateShortcutCandidate(map, reinterpret_cast<HeapObject**>(slot),
224 reinterpret_cast<ConsString*>(source), size);
225 break;
226 default:
227 EvacuateObjectDefault(map, slot, source, size);
228 break;
229 }
230 }
231
ScavengeObject(HeapObjectReference ** p,HeapObject * object)232 void Scavenger::ScavengeObject(HeapObjectReference** p, HeapObject* object) {
233 DCHECK(Heap::InFromSpace(object));
234
235 // Synchronized load that consumes the publishing CAS of MigrateObject.
236 MapWord first_word = object->synchronized_map_word();
237
238 // If the first word is a forwarding address, the object has already been
239 // copied.
240 if (first_word.IsForwardingAddress()) {
241 HeapObject* dest = first_word.ToForwardingAddress();
242 DCHECK(Heap::InFromSpace(*p));
243 if ((*p)->IsWeakHeapObject()) {
244 *p = HeapObjectReference::Weak(dest);
245 } else {
246 DCHECK((*p)->IsStrongHeapObject());
247 *p = HeapObjectReference::Strong(dest);
248 }
249 return;
250 }
251
252 Map* map = first_word.ToMap();
253 // AllocationMementos are unrooted and shouldn't survive a scavenge
254 DCHECK_NE(ReadOnlyRoots(heap()).allocation_memento_map(), map);
255 // Call the slow part of scavenge object.
256 EvacuateObject(p, map, object);
257 }
258
CheckAndScavengeObject(Heap * heap,Address slot_address)259 SlotCallbackResult Scavenger::CheckAndScavengeObject(Heap* heap,
260 Address slot_address) {
261 MaybeObject** slot = reinterpret_cast<MaybeObject**>(slot_address);
262 MaybeObject* object = *slot;
263 if (Heap::InFromSpace(object)) {
264 HeapObject* heap_object;
265 bool success = object->ToStrongOrWeakHeapObject(&heap_object);
266 USE(success);
267 DCHECK(success);
268 DCHECK(heap_object->IsHeapObject());
269
270 ScavengeObject(reinterpret_cast<HeapObjectReference**>(slot), heap_object);
271
272 object = *slot;
273 // If the object was in from space before and is after executing the
274 // callback in to space, the object is still live.
275 // Unfortunately, we do not know about the slot. It could be in a
276 // just freed free space object.
277 PageMemoryFence(object);
278 if (Heap::InToSpace(object)) {
279 return KEEP_SLOT;
280 }
281 } else if (Heap::InToSpace(object)) {
282 // Already updated slot. This can happen when processing of the work list
283 // is interleaved with processing roots.
284 return KEEP_SLOT;
285 }
286 // Slots can point to "to" space if the slot has been recorded multiple
287 // times in the remembered set. We remove the redundant slot now.
288 return REMOVE_SLOT;
289 }
290
VisitPointers(HeapObject * host,Object ** start,Object ** end)291 void ScavengeVisitor::VisitPointers(HeapObject* host, Object** start,
292 Object** end) {
293 for (Object** p = start; p < end; p++) {
294 Object* object = *p;
295 if (!Heap::InNewSpace(object)) continue;
296 scavenger_->ScavengeObject(reinterpret_cast<HeapObjectReference**>(p),
297 reinterpret_cast<HeapObject*>(object));
298 }
299 }
300
VisitPointers(HeapObject * host,MaybeObject ** start,MaybeObject ** end)301 void ScavengeVisitor::VisitPointers(HeapObject* host, MaybeObject** start,
302 MaybeObject** end) {
303 for (MaybeObject** p = start; p < end; p++) {
304 MaybeObject* object = *p;
305 if (!Heap::InNewSpace(object)) continue;
306 // Treat the weak reference as strong.
307 HeapObject* heap_object;
308 if (object->ToStrongOrWeakHeapObject(&heap_object)) {
309 scavenger_->ScavengeObject(reinterpret_cast<HeapObjectReference**>(p),
310 heap_object);
311 } else {
312 UNREACHABLE();
313 }
314 }
315 }
316
317 } // namespace internal
318 } // namespace v8
319
320 #endif // V8_HEAP_SCAVENGER_INL_H_
321