1 // Copyright 2018 the V8 project authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "src/snapshot/embedded/embedded-data.h"
6
7 #include "src/codegen/assembler-inl.h"
8 #include "src/codegen/callable.h"
9 #include "src/objects/objects-inl.h"
10 #include "src/snapshot/snapshot-utils.h"
11 #include "src/snapshot/snapshot.h"
12
13 namespace v8 {
14 namespace internal {
15
16 // static
PcIsOffHeap(Isolate * isolate,Address pc)17 bool InstructionStream::PcIsOffHeap(Isolate* isolate, Address pc) {
18 const Address start =
19 reinterpret_cast<Address>(isolate->embedded_blob_code());
20 return start <= pc && pc < start + isolate->embedded_blob_code_size();
21 }
22
23 // static
TryLookupCode(Isolate * isolate,Address address)24 Code InstructionStream::TryLookupCode(Isolate* isolate, Address address) {
25 if (!PcIsOffHeap(isolate, address)) return Code();
26
27 EmbeddedData d = EmbeddedData::FromBlob();
28 if (address < d.InstructionStartOfBuiltin(0)) return Code();
29
30 // Note: Addresses within the padding section between builtins (i.e. within
31 // start + size <= address < start + padded_size) are interpreted as belonging
32 // to the preceding builtin.
33
34 int l = 0, r = Builtins::builtin_count;
35 while (l < r) {
36 const int mid = (l + r) / 2;
37 Address start = d.InstructionStartOfBuiltin(mid);
38 Address end = start + d.PaddedInstructionSizeOfBuiltin(mid);
39
40 if (address < start) {
41 r = mid;
42 } else if (address >= end) {
43 l = mid + 1;
44 } else {
45 return isolate->builtins()->builtin(mid);
46 }
47 }
48
49 UNREACHABLE();
50 }
51
52 // static
CreateOffHeapInstructionStream(Isolate * isolate,uint8_t ** code,uint32_t * code_size,uint8_t ** data,uint32_t * data_size)53 void InstructionStream::CreateOffHeapInstructionStream(Isolate* isolate,
54 uint8_t** code,
55 uint32_t* code_size,
56 uint8_t** data,
57 uint32_t* data_size) {
58 // Create the embedded blob from scratch using the current Isolate's heap.
59 EmbeddedData d = EmbeddedData::FromIsolate(isolate);
60
61 // Allocate the backing store that will contain the embedded blob in this
62 // Isolate. The backing store is on the native heap, *not* on V8's garbage-
63 // collected heap.
64 v8::PageAllocator* page_allocator = v8::internal::GetPlatformPageAllocator();
65 const uint32_t alignment =
66 static_cast<uint32_t>(page_allocator->AllocatePageSize());
67
68 void* const requested_allocation_code_address =
69 AlignedAddress(isolate->heap()->GetRandomMmapAddr(), alignment);
70 const uint32_t allocation_code_size = RoundUp(d.code_size(), alignment);
71 uint8_t* allocated_code_bytes = static_cast<uint8_t*>(AllocatePages(
72 page_allocator, requested_allocation_code_address, allocation_code_size,
73 alignment, PageAllocator::kReadWrite));
74 CHECK_NOT_NULL(allocated_code_bytes);
75
76 void* const requested_allocation_data_address =
77 AlignedAddress(isolate->heap()->GetRandomMmapAddr(), alignment);
78 const uint32_t allocation_data_size = RoundUp(d.data_size(), alignment);
79 uint8_t* allocated_data_bytes = static_cast<uint8_t*>(AllocatePages(
80 page_allocator, requested_allocation_data_address, allocation_data_size,
81 alignment, PageAllocator::kReadWrite));
82 CHECK_NOT_NULL(allocated_data_bytes);
83
84 // Copy the embedded blob into the newly allocated backing store. Switch
85 // permissions to read-execute since builtin code is immutable from now on
86 // and must be executable in case any JS execution is triggered.
87 //
88 // Once this backing store is set as the current_embedded_blob, V8 cannot tell
89 // the difference between a 'real' embedded build (where the blob is embedded
90 // in the binary) and what we are currently setting up here (where the blob is
91 // on the native heap).
92 std::memcpy(allocated_code_bytes, d.code(), d.code_size());
93 CHECK(SetPermissions(page_allocator, allocated_code_bytes,
94 allocation_code_size, PageAllocator::kReadExecute));
95
96 std::memcpy(allocated_data_bytes, d.data(), d.data_size());
97 CHECK(SetPermissions(page_allocator, allocated_data_bytes,
98 allocation_data_size, PageAllocator::kRead));
99
100 *code = allocated_code_bytes;
101 *code_size = d.code_size();
102 *data = allocated_data_bytes;
103 *data_size = d.data_size();
104
105 d.Dispose();
106 }
107
108 // static
FreeOffHeapInstructionStream(uint8_t * code,uint32_t code_size,uint8_t * data,uint32_t data_size)109 void InstructionStream::FreeOffHeapInstructionStream(uint8_t* code,
110 uint32_t code_size,
111 uint8_t* data,
112 uint32_t data_size) {
113 v8::PageAllocator* page_allocator = v8::internal::GetPlatformPageAllocator();
114 const uint32_t page_size =
115 static_cast<uint32_t>(page_allocator->AllocatePageSize());
116 CHECK(FreePages(page_allocator, code, RoundUp(code_size, page_size)));
117 CHECK(FreePages(page_allocator, data, RoundUp(data_size, page_size)));
118 }
119
120 namespace {
121
BuiltinAliasesOffHeapTrampolineRegister(Isolate * isolate,Code code)122 bool BuiltinAliasesOffHeapTrampolineRegister(Isolate* isolate, Code code) {
123 DCHECK(Builtins::IsIsolateIndependent(code.builtin_index()));
124 switch (Builtins::KindOf(code.builtin_index())) {
125 case Builtins::CPP:
126 case Builtins::TFC:
127 case Builtins::TFH:
128 case Builtins::TFJ:
129 case Builtins::TFS:
130 break;
131
132 // Bytecode handlers will only ever be used by the interpreter and so there
133 // will never be a need to use trampolines with them.
134 case Builtins::BCH:
135 case Builtins::ASM:
136 // TODO(jgruber): Extend checks to remaining kinds.
137 return false;
138 }
139
140 Callable callable = Builtins::CallableFor(
141 isolate, static_cast<Builtins::Name>(code.builtin_index()));
142 CallInterfaceDescriptor descriptor = callable.descriptor();
143
144 if (descriptor.ContextRegister() == kOffHeapTrampolineRegister) {
145 return true;
146 }
147
148 for (int i = 0; i < descriptor.GetRegisterParameterCount(); i++) {
149 Register reg = descriptor.GetRegisterParameter(i);
150 if (reg == kOffHeapTrampolineRegister) return true;
151 }
152
153 return false;
154 }
155
FinalizeEmbeddedCodeTargets(Isolate * isolate,EmbeddedData * blob)156 void FinalizeEmbeddedCodeTargets(Isolate* isolate, EmbeddedData* blob) {
157 static const int kRelocMask =
158 RelocInfo::ModeMask(RelocInfo::CODE_TARGET) |
159 RelocInfo::ModeMask(RelocInfo::RELATIVE_CODE_TARGET);
160
161 STATIC_ASSERT(Builtins::kAllBuiltinsAreIsolateIndependent);
162 for (int i = 0; i < Builtins::builtin_count; i++) {
163 Code code = isolate->builtins()->builtin(i);
164 RelocIterator on_heap_it(code, kRelocMask);
165 RelocIterator off_heap_it(blob, code, kRelocMask);
166
167 #if defined(V8_TARGET_ARCH_X64) || defined(V8_TARGET_ARCH_ARM64) || \
168 defined(V8_TARGET_ARCH_ARM) || defined(V8_TARGET_ARCH_MIPS) || \
169 defined(V8_TARGET_ARCH_IA32) || defined(V8_TARGET_ARCH_S390)
170 // On these platforms we emit relative builtin-to-builtin
171 // jumps for isolate independent builtins in the snapshot. This fixes up the
172 // relative jumps to the right offsets in the snapshot.
173 // See also: Code::IsIsolateIndependent.
174 while (!on_heap_it.done()) {
175 DCHECK(!off_heap_it.done());
176
177 RelocInfo* rinfo = on_heap_it.rinfo();
178 DCHECK_EQ(rinfo->rmode(), off_heap_it.rinfo()->rmode());
179 Code target = Code::GetCodeFromTargetAddress(rinfo->target_address());
180 CHECK(Builtins::IsIsolateIndependentBuiltin(target));
181
182 // Do not emit write-barrier for off-heap writes.
183 off_heap_it.rinfo()->set_target_address(
184 blob->InstructionStartOfBuiltin(target.builtin_index()),
185 SKIP_WRITE_BARRIER);
186
187 on_heap_it.next();
188 off_heap_it.next();
189 }
190 DCHECK(off_heap_it.done());
191 #else
192 // Architectures other than x64 and arm/arm64 do not use pc-relative calls
193 // and thus must not contain embedded code targets. Instead, we use an
194 // indirection through the root register.
195 CHECK(on_heap_it.done());
196 CHECK(off_heap_it.done());
197 #endif // defined(V8_TARGET_ARCH_X64) || defined(V8_TARGET_ARCH_ARM64)
198 }
199 }
200
201 } // namespace
202
203 // static
FromIsolate(Isolate * isolate)204 EmbeddedData EmbeddedData::FromIsolate(Isolate* isolate) {
205 Builtins* builtins = isolate->builtins();
206
207 // Store instruction stream lengths and offsets.
208 std::vector<struct LayoutDescription> layout_descriptions(kTableSize);
209
210 bool saw_unsafe_builtin = false;
211 uint32_t raw_code_size = 0;
212 uint32_t raw_data_size = 0;
213 STATIC_ASSERT(Builtins::kAllBuiltinsAreIsolateIndependent);
214 for (int i = 0; i < Builtins::builtin_count; i++) {
215 Code code = builtins->builtin(i);
216
217 // Sanity-check that the given builtin is isolate-independent and does not
218 // use the trampoline register in its calling convention.
219 if (!code.IsIsolateIndependent(isolate)) {
220 saw_unsafe_builtin = true;
221 fprintf(stderr, "%s is not isolate-independent.\n", Builtins::name(i));
222 }
223 if (BuiltinAliasesOffHeapTrampolineRegister(isolate, code)) {
224 saw_unsafe_builtin = true;
225 fprintf(stderr, "%s aliases the off-heap trampoline register.\n",
226 Builtins::name(i));
227 }
228
229 uint32_t instruction_size =
230 static_cast<uint32_t>(code.raw_instruction_size());
231 uint32_t metadata_size = static_cast<uint32_t>(code.raw_metadata_size());
232
233 DCHECK_EQ(0, raw_code_size % kCodeAlignment);
234 layout_descriptions[i].instruction_offset = raw_code_size;
235 layout_descriptions[i].instruction_length = instruction_size;
236 layout_descriptions[i].metadata_offset = raw_data_size;
237 layout_descriptions[i].metadata_length = metadata_size;
238
239 // Align the start of each section.
240 raw_code_size += PadAndAlignCode(instruction_size);
241 raw_data_size += PadAndAlignData(metadata_size);
242 }
243 CHECK_WITH_MSG(
244 !saw_unsafe_builtin,
245 "One or more builtins marked as isolate-independent either contains "
246 "isolate-dependent code or aliases the off-heap trampoline register. "
247 "If in doubt, ask jgruber@");
248
249 // Allocate space for the code section, value-initialized to 0.
250 STATIC_ASSERT(RawCodeOffset() == 0);
251 const uint32_t blob_code_size = RawCodeOffset() + raw_code_size;
252 uint8_t* const blob_code = new uint8_t[blob_code_size]();
253
254 // Allocate space for the data section, value-initialized to 0.
255 STATIC_ASSERT(IsAligned(FixedDataSize(), Code::kMetadataAlignment));
256 const uint32_t blob_data_size = FixedDataSize() + raw_data_size;
257 uint8_t* const blob_data = new uint8_t[blob_data_size]();
258
259 // Initially zap the entire blob, effectively padding the alignment area
260 // between two builtins with int3's (on x64/ia32).
261 ZapCode(reinterpret_cast<Address>(blob_code), blob_code_size);
262
263 // Hash relevant parts of the Isolate's heap and store the result.
264 {
265 STATIC_ASSERT(IsolateHashSize() == kSizetSize);
266 const size_t hash = isolate->HashIsolateForEmbeddedBlob();
267 std::memcpy(blob_data + IsolateHashOffset(), &hash, IsolateHashSize());
268 }
269
270 // Write the layout_descriptions tables.
271 DCHECK_EQ(LayoutDescriptionTableSize(),
272 sizeof(layout_descriptions[0]) * layout_descriptions.size());
273 std::memcpy(blob_data + LayoutDescriptionTableOffset(),
274 layout_descriptions.data(), LayoutDescriptionTableSize());
275
276 // .. and the variable-size data section.
277 uint8_t* const raw_metadata_start = blob_data + RawMetadataOffset();
278 STATIC_ASSERT(Builtins::kAllBuiltinsAreIsolateIndependent);
279 for (int i = 0; i < Builtins::builtin_count; i++) {
280 Code code = builtins->builtin(i);
281 uint32_t offset = layout_descriptions[i].metadata_offset;
282 uint8_t* dst = raw_metadata_start + offset;
283 DCHECK_LE(RawMetadataOffset() + offset + code.raw_metadata_size(),
284 blob_data_size);
285 std::memcpy(dst, reinterpret_cast<uint8_t*>(code.raw_metadata_start()),
286 code.raw_metadata_size());
287 }
288
289 // .. and the variable-size code section.
290 uint8_t* const raw_code_start = blob_code + RawCodeOffset();
291 STATIC_ASSERT(Builtins::kAllBuiltinsAreIsolateIndependent);
292 for (int i = 0; i < Builtins::builtin_count; i++) {
293 Code code = builtins->builtin(i);
294 uint32_t offset = layout_descriptions[i].instruction_offset;
295 uint8_t* dst = raw_code_start + offset;
296 DCHECK_LE(RawCodeOffset() + offset + code.raw_instruction_size(),
297 blob_code_size);
298 std::memcpy(dst, reinterpret_cast<uint8_t*>(code.raw_instruction_start()),
299 code.raw_instruction_size());
300 }
301
302 EmbeddedData d(blob_code, blob_code_size, blob_data, blob_data_size);
303
304 // Fix up call targets that point to other embedded builtins.
305 FinalizeEmbeddedCodeTargets(isolate, &d);
306
307 // Hash the blob and store the result.
308 {
309 STATIC_ASSERT(EmbeddedBlobDataHashSize() == kSizetSize);
310 const size_t data_hash = d.CreateEmbeddedBlobDataHash();
311 std::memcpy(blob_data + EmbeddedBlobDataHashOffset(), &data_hash,
312 EmbeddedBlobDataHashSize());
313
314 STATIC_ASSERT(EmbeddedBlobCodeHashSize() == kSizetSize);
315 const size_t code_hash = d.CreateEmbeddedBlobCodeHash();
316 std::memcpy(blob_data + EmbeddedBlobCodeHashOffset(), &code_hash,
317 EmbeddedBlobCodeHashSize());
318
319 DCHECK_EQ(data_hash, d.CreateEmbeddedBlobDataHash());
320 DCHECK_EQ(data_hash, d.EmbeddedBlobDataHash());
321 DCHECK_EQ(code_hash, d.CreateEmbeddedBlobCodeHash());
322 DCHECK_EQ(code_hash, d.EmbeddedBlobCodeHash());
323 }
324
325 if (FLAG_serialization_statistics) d.PrintStatistics();
326
327 return d;
328 }
329
InstructionStartOfBuiltin(int i) const330 Address EmbeddedData::InstructionStartOfBuiltin(int i) const {
331 DCHECK(Builtins::IsBuiltinId(i));
332 const struct LayoutDescription* descs = LayoutDescription();
333 const uint8_t* result = RawCode() + descs[i].instruction_offset;
334 DCHECK_LT(result, code_ + code_size_);
335 return reinterpret_cast<Address>(result);
336 }
337
InstructionSizeOfBuiltin(int i) const338 uint32_t EmbeddedData::InstructionSizeOfBuiltin(int i) const {
339 DCHECK(Builtins::IsBuiltinId(i));
340 const struct LayoutDescription* descs = LayoutDescription();
341 return descs[i].instruction_length;
342 }
343
MetadataStartOfBuiltin(int i) const344 Address EmbeddedData::MetadataStartOfBuiltin(int i) const {
345 DCHECK(Builtins::IsBuiltinId(i));
346 const struct LayoutDescription* descs = LayoutDescription();
347 const uint8_t* result = RawMetadata() + descs[i].metadata_offset;
348 DCHECK_LE(descs[i].metadata_offset, data_size_);
349 return reinterpret_cast<Address>(result);
350 }
351
MetadataSizeOfBuiltin(int i) const352 uint32_t EmbeddedData::MetadataSizeOfBuiltin(int i) const {
353 DCHECK(Builtins::IsBuiltinId(i));
354 const struct LayoutDescription* descs = LayoutDescription();
355 return descs[i].metadata_length;
356 }
357
InstructionStartOfBytecodeHandlers() const358 Address EmbeddedData::InstructionStartOfBytecodeHandlers() const {
359 return InstructionStartOfBuiltin(Builtins::kFirstBytecodeHandler);
360 }
361
InstructionEndOfBytecodeHandlers() const362 Address EmbeddedData::InstructionEndOfBytecodeHandlers() const {
363 STATIC_ASSERT(Builtins::kFirstBytecodeHandler + kNumberOfBytecodeHandlers +
364 2 * kNumberOfWideBytecodeHandlers ==
365 Builtins::builtin_count);
366 int lastBytecodeHandler = Builtins::builtin_count - 1;
367 return InstructionStartOfBuiltin(lastBytecodeHandler) +
368 InstructionSizeOfBuiltin(lastBytecodeHandler);
369 }
370
CreateEmbeddedBlobDataHash() const371 size_t EmbeddedData::CreateEmbeddedBlobDataHash() const {
372 STATIC_ASSERT(EmbeddedBlobDataHashOffset() == 0);
373 STATIC_ASSERT(EmbeddedBlobCodeHashOffset() == EmbeddedBlobDataHashSize());
374 STATIC_ASSERT(IsolateHashOffset() ==
375 EmbeddedBlobCodeHashOffset() + EmbeddedBlobCodeHashSize());
376 static constexpr uint32_t kFirstHashedDataOffset = IsolateHashOffset();
377 // Hash the entire data section except the embedded blob hash fields
378 // themselves.
379 Vector<const byte> payload(data_ + kFirstHashedDataOffset,
380 data_size_ - kFirstHashedDataOffset);
381 return Checksum(payload);
382 }
383
CreateEmbeddedBlobCodeHash() const384 size_t EmbeddedData::CreateEmbeddedBlobCodeHash() const {
385 CHECK(FLAG_text_is_readable);
386 Vector<const byte> payload(code_, code_size_);
387 return Checksum(payload);
388 }
389
PrintStatistics() const390 void EmbeddedData::PrintStatistics() const {
391 DCHECK(FLAG_serialization_statistics);
392
393 constexpr int kCount = Builtins::builtin_count;
394 int sizes[kCount];
395 STATIC_ASSERT(Builtins::kAllBuiltinsAreIsolateIndependent);
396 for (int i = 0; i < kCount; i++) {
397 sizes[i] = InstructionSizeOfBuiltin(i);
398 }
399
400 // Sort for percentiles.
401 std::sort(&sizes[0], &sizes[kCount]);
402
403 const int k50th = kCount * 0.5;
404 const int k75th = kCount * 0.75;
405 const int k90th = kCount * 0.90;
406 const int k99th = kCount * 0.99;
407
408 PrintF("EmbeddedData:\n");
409 PrintF(" Total size: %d\n",
410 static_cast<int>(code_size() + data_size()));
411 PrintF(" Data size: %d\n",
412 static_cast<int>(data_size()));
413 PrintF(" Code size: %d\n", static_cast<int>(code_size()));
414 PrintF(" Instruction size (50th percentile): %d\n", sizes[k50th]);
415 PrintF(" Instruction size (75th percentile): %d\n", sizes[k75th]);
416 PrintF(" Instruction size (90th percentile): %d\n", sizes[k90th]);
417 PrintF(" Instruction size (99th percentile): %d\n", sizes[k99th]);
418 PrintF("\n");
419 }
420
421 } // namespace internal
422 } // namespace v8
423