1 // Copyright 2020 The Chromium Authors
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "base/allocator/partition_allocator/partition_address_space.h"
6
7 #include <array>
8 #include <cstddef>
9 #include <cstdint>
10 #include <ostream>
11 #include <string>
12
13 #include "base/allocator/partition_allocator/address_pool_manager.h"
14 #include "base/allocator/partition_allocator/compressed_pointer.h"
15 #include "base/allocator/partition_allocator/page_allocator.h"
16 #include "base/allocator/partition_allocator/partition_alloc_base/bits.h"
17 #include "base/allocator/partition_allocator/partition_alloc_base/compiler_specific.h"
18 #include "base/allocator/partition_allocator/partition_alloc_base/debug/alias.h"
19 #include "base/allocator/partition_allocator/partition_alloc_buildflags.h"
20 #include "base/allocator/partition_allocator/partition_alloc_check.h"
21 #include "base/allocator/partition_allocator/partition_alloc_config.h"
22 #include "base/allocator/partition_allocator/partition_alloc_constants.h"
23 #include "base/allocator/partition_allocator/pkey.h"
24 #include "build/build_config.h"
25
26 #if BUILDFLAG(IS_IOS)
27 #include <mach-o/dyld.h>
28 #endif
29
30 #if BUILDFLAG(IS_WIN)
31 #include <windows.h>
32 #endif // BUILDFLAG(IS_WIN)
33
34 #if PA_CONFIG(ENABLE_SHADOW_METADATA) || BUILDFLAG(ENABLE_PKEYS)
35 #include <sys/mman.h>
36 #endif
37
38 namespace partition_alloc::internal {
39
40 #if BUILDFLAG(HAS_64_BIT_POINTERS)
41
42 namespace {
43
44 #if BUILDFLAG(IS_WIN)
45
HandlePoolAllocFailureOutOfVASpace()46 PA_NOINLINE void HandlePoolAllocFailureOutOfVASpace() {
47 PA_NO_CODE_FOLDING();
48 PA_CHECK(false);
49 }
50
HandlePoolAllocFailureOutOfCommitCharge()51 PA_NOINLINE void HandlePoolAllocFailureOutOfCommitCharge() {
52 PA_NO_CODE_FOLDING();
53 PA_CHECK(false);
54 }
55 #endif // BUILDFLAG(IS_WIN)
56
HandlePoolAllocFailure()57 PA_NOINLINE void HandlePoolAllocFailure() {
58 PA_NO_CODE_FOLDING();
59 uint32_t alloc_page_error_code = GetAllocPageErrorCode();
60 PA_DEBUG_DATA_ON_STACK("error", static_cast<size_t>(alloc_page_error_code));
61 // It's important to easily differentiate these two failures on Windows, so
62 // crash with different stacks.
63 #if BUILDFLAG(IS_WIN)
64 if (alloc_page_error_code == ERROR_NOT_ENOUGH_MEMORY) {
65 // The error code says NOT_ENOUGH_MEMORY, but since we only do MEM_RESERVE,
66 // it must be VA space exhaustion.
67 HandlePoolAllocFailureOutOfVASpace();
68 } else if (alloc_page_error_code == ERROR_COMMITMENT_LIMIT) {
69 // Should not happen, since as of Windows 8.1+, reserving address space
70 // should not be charged against the commit limit, aside from a very small
71 // amount per 64kiB block. Keep this path anyway, to check in crash reports.
72 HandlePoolAllocFailureOutOfCommitCharge();
73 } else
74 #endif // BUILDFLAG(IS_WIN)
75 {
76 PA_CHECK(false);
77 }
78 }
79
80 } // namespace
81
82 #if BUILDFLAG(ENABLE_PKEYS)
83 alignas(PA_PKEY_ALIGN_SZ)
84 #else
85 alignas(kPartitionCachelineSize)
86 #endif
87 PartitionAddressSpace::PoolSetup PartitionAddressSpace::setup_;
88
89 #if PA_CONFIG(ENABLE_SHADOW_METADATA)
90 std::ptrdiff_t PartitionAddressSpace::regular_pool_shadow_offset_ = 0;
91 std::ptrdiff_t PartitionAddressSpace::brp_pool_shadow_offset_ = 0;
92 #endif
93
94 #if PA_CONFIG(DYNAMICALLY_SELECT_POOL_SIZE)
95 #if !BUILDFLAG(IS_IOS)
96 #error Dynamic pool size is only supported on iOS.
97 #endif
98
99 namespace {
IsIOSTestProcess()100 bool IsIOSTestProcess() {
101 // On iOS, only applications with the extended virtual addressing entitlement
102 // can use a large address space. Since Earl Grey test runner apps cannot get
103 // entitlements, they must use a much smaller pool size. Similarly,
104 // integration tests for ChromeWebView end up with two PartitionRoots since
105 // both the integration tests and ChromeWebView have a copy of base/. Even
106 // with the entitlement, there is insufficient address space for two
107 // PartitionRoots, so a smaller pool size is needed.
108
109 // Use a fixed buffer size to avoid allocation inside the allocator.
110 constexpr size_t path_buffer_size = 8192;
111 char executable_path[path_buffer_size];
112
113 uint32_t executable_length = path_buffer_size;
114 int rv = _NSGetExecutablePath(executable_path, &executable_length);
115 PA_CHECK(!rv);
116 size_t executable_path_length =
117 std::char_traits<char>::length(executable_path);
118
119 auto has_suffix = [&](const char* suffix) -> bool {
120 size_t suffix_length = std::char_traits<char>::length(suffix);
121 if (executable_path_length < suffix_length) {
122 return false;
123 }
124 return std::char_traits<char>::compare(
125 executable_path + (executable_path_length - suffix_length),
126 suffix, suffix_length) == 0;
127 };
128
129 return has_suffix("Runner") || has_suffix("ios_web_view_inttests");
130 }
131 } // namespace
132
RegularPoolSize()133 PA_ALWAYS_INLINE size_t PartitionAddressSpace::RegularPoolSize() {
134 return IsIOSTestProcess() ? kRegularPoolSizeForIOSTestProcess
135 : kRegularPoolSize;
136 }
BRPPoolSize()137 PA_ALWAYS_INLINE size_t PartitionAddressSpace::BRPPoolSize() {
138 return IsIOSTestProcess() ? kBRPPoolSizeForIOSTestProcess : kBRPPoolSize;
139 }
140 #endif // PA_CONFIG(DYNAMICALLY_SELECT_POOL_SIZE)
141
Init()142 void PartitionAddressSpace::Init() {
143 if (IsInitialized()) {
144 return;
145 }
146
147 size_t regular_pool_size = RegularPoolSize();
148 size_t brp_pool_size = BRPPoolSize();
149
150 #if PA_CONFIG(GLUE_CORE_POOLS)
151 // Gluing core pools (regular & BRP) makes sense only when both pools are of
152 // the same size. This the only way we can check belonging to either of the
153 // two with a single bitmask operation.
154 PA_CHECK(regular_pool_size == brp_pool_size);
155
156 // TODO(crbug.com/1362969): Support PA_ENABLE_SHADOW_METADATA.
157 int pools_fd = -1;
158
159 size_t glued_pool_sizes = regular_pool_size * 2;
160 // Note, BRP pool requires to be preceded by a "forbidden zone", which is
161 // conveniently taken care of by the last guard page of the regular pool.
162 setup_.regular_pool_base_address_ =
163 AllocPages(glued_pool_sizes, glued_pool_sizes,
164 PageAccessibilityConfiguration(
165 PageAccessibilityConfiguration::kInaccessible),
166 PageTag::kPartitionAlloc, pools_fd);
167 if (!setup_.regular_pool_base_address_) {
168 HandlePoolAllocFailure();
169 }
170 setup_.brp_pool_base_address_ =
171 setup_.regular_pool_base_address_ + regular_pool_size;
172 #else // PA_CONFIG(GLUE_CORE_POOLS)
173 #if PA_CONFIG(ENABLE_SHADOW_METADATA)
174 int regular_pool_fd = memfd_create("/regular_pool", MFD_CLOEXEC);
175 #else
176 int regular_pool_fd = -1;
177 #endif
178 setup_.regular_pool_base_address_ =
179 AllocPages(regular_pool_size, regular_pool_size,
180 PageAccessibilityConfiguration(
181 PageAccessibilityConfiguration::kInaccessible),
182 PageTag::kPartitionAlloc, regular_pool_fd);
183 if (!setup_.regular_pool_base_address_) {
184 HandlePoolAllocFailure();
185 }
186
187 #if PA_CONFIG(ENABLE_SHADOW_METADATA)
188 int brp_pool_fd = memfd_create("/brp_pool", MFD_CLOEXEC);
189 #else
190 int brp_pool_fd = -1;
191 #endif
192 // Reserve an extra allocation granularity unit before the BRP pool, but keep
193 // the pool aligned at BRPPoolSize(). A pointer immediately past an allocation
194 // is a valid pointer, and having a "forbidden zone" before the BRP pool
195 // prevents such a pointer from "sneaking into" the pool.
196 const size_t kForbiddenZoneSize = PageAllocationGranularity();
197 uintptr_t base_address = AllocPagesWithAlignOffset(
198 0, brp_pool_size + kForbiddenZoneSize, brp_pool_size,
199 brp_pool_size - kForbiddenZoneSize,
200 PageAccessibilityConfiguration(
201 PageAccessibilityConfiguration::kInaccessible),
202 PageTag::kPartitionAlloc, brp_pool_fd);
203 if (!base_address) {
204 HandlePoolAllocFailure();
205 }
206 setup_.brp_pool_base_address_ = base_address + kForbiddenZoneSize;
207 #endif // PA_CONFIG(GLUE_CORE_POOLS)
208
209 #if PA_CONFIG(DYNAMICALLY_SELECT_POOL_SIZE)
210 setup_.regular_pool_base_mask_ = ~(regular_pool_size - 1);
211 setup_.brp_pool_base_mask_ = ~(brp_pool_size - 1);
212 #if PA_CONFIG(GLUE_CORE_POOLS)
213 // When PA_GLUE_CORE_POOLS is on, the BRP pool is placed at the end of the
214 // regular pool, effectively forming one virtual pool of a twice bigger
215 // size. Adjust the mask appropriately.
216 setup_.core_pools_base_mask_ = setup_.regular_pool_base_mask_ << 1;
217 PA_DCHECK(setup_.core_pools_base_mask_ == (setup_.brp_pool_base_mask_ << 1));
218 #endif
219 #endif // PA_CONFIG(DYNAMICALLY_SELECT_POOL_SIZE)
220
221 AddressPoolManager::GetInstance().Add(
222 kRegularPoolHandle, setup_.regular_pool_base_address_, regular_pool_size);
223 AddressPoolManager::GetInstance().Add(
224 kBRPPoolHandle, setup_.brp_pool_base_address_, brp_pool_size);
225
226 // Sanity check pool alignment.
227 PA_DCHECK(!(setup_.regular_pool_base_address_ & (regular_pool_size - 1)));
228 PA_DCHECK(!(setup_.brp_pool_base_address_ & (brp_pool_size - 1)));
229 #if PA_CONFIG(GLUE_CORE_POOLS)
230 PA_DCHECK(!(setup_.regular_pool_base_address_ & (glued_pool_sizes - 1)));
231 #endif
232
233 // Sanity check pool belonging.
234 PA_DCHECK(!IsInRegularPool(setup_.regular_pool_base_address_ - 1));
235 PA_DCHECK(IsInRegularPool(setup_.regular_pool_base_address_));
236 PA_DCHECK(IsInRegularPool(setup_.regular_pool_base_address_ +
237 regular_pool_size - 1));
238 PA_DCHECK(
239 !IsInRegularPool(setup_.regular_pool_base_address_ + regular_pool_size));
240 PA_DCHECK(!IsInBRPPool(setup_.brp_pool_base_address_ - 1));
241 PA_DCHECK(IsInBRPPool(setup_.brp_pool_base_address_));
242 PA_DCHECK(IsInBRPPool(setup_.brp_pool_base_address_ + brp_pool_size - 1));
243 PA_DCHECK(!IsInBRPPool(setup_.brp_pool_base_address_ + brp_pool_size));
244 #if PA_CONFIG(GLUE_CORE_POOLS)
245 PA_DCHECK(!IsInCorePools(setup_.regular_pool_base_address_ - 1));
246 PA_DCHECK(IsInCorePools(setup_.regular_pool_base_address_));
247 PA_DCHECK(
248 IsInCorePools(setup_.regular_pool_base_address_ + regular_pool_size - 1));
249 PA_DCHECK(
250 IsInCorePools(setup_.regular_pool_base_address_ + regular_pool_size));
251 PA_DCHECK(IsInCorePools(setup_.brp_pool_base_address_ - 1));
252 PA_DCHECK(IsInCorePools(setup_.brp_pool_base_address_));
253 PA_DCHECK(IsInCorePools(setup_.brp_pool_base_address_ + brp_pool_size - 1));
254 PA_DCHECK(!IsInCorePools(setup_.brp_pool_base_address_ + brp_pool_size));
255 #endif // PA_CONFIG(GLUE_CORE_POOLS)
256
257 #if PA_CONFIG(STARSCAN_USE_CARD_TABLE)
258 // Reserve memory for PCScan quarantine card table.
259 uintptr_t requested_address = setup_.regular_pool_base_address_;
260 uintptr_t actual_address = AddressPoolManager::GetInstance().Reserve(
261 kRegularPoolHandle, requested_address, kSuperPageSize);
262 PA_CHECK(requested_address == actual_address)
263 << "QuarantineCardTable is required to be allocated at the beginning of "
264 "the regular pool";
265 #endif // PA_CONFIG(STARSCAN_USE_CARD_TABLE)
266
267 #if PA_CONFIG(ENABLE_SHADOW_METADATA)
268 // Reserve memory for the shadow pools.
269 uintptr_t regular_pool_shadow_address =
270 AllocPages(regular_pool_size, regular_pool_size,
271 PageAccessibilityConfiguration(
272 PageAccessibilityConfiguration::kInaccessible),
273 PageTag::kPartitionAlloc, regular_pool_fd);
274 regular_pool_shadow_offset_ =
275 regular_pool_shadow_address - setup_.regular_pool_base_address_;
276
277 uintptr_t brp_pool_shadow_address = AllocPagesWithAlignOffset(
278 0, brp_pool_size + kForbiddenZoneSize, brp_pool_size,
279 brp_pool_size - kForbiddenZoneSize,
280 PageAccessibilityConfiguration(
281 PageAccessibilityConfiguration::kInaccessible),
282 PageTag::kPartitionAlloc, brp_pool_fd);
283 brp_pool_shadow_offset_ =
284 brp_pool_shadow_address - setup_.brp_pool_base_address_;
285 #endif
286
287 #if PA_CONFIG(POINTER_COMPRESSION)
288 CompressedPointerBaseGlobal::SetBase(setup_.regular_pool_base_address_);
289 #endif // PA_CONFIG(POINTER_COMPRESSION)
290 }
291
InitConfigurablePool(uintptr_t pool_base,size_t size)292 void PartitionAddressSpace::InitConfigurablePool(uintptr_t pool_base,
293 size_t size) {
294 // The ConfigurablePool must only be initialized once.
295 PA_CHECK(!IsConfigurablePoolInitialized());
296
297 #if BUILDFLAG(ENABLE_PKEYS)
298 // It's possible that the pkey pool has been initialized first, in which case
299 // the setup_ memory has been made read-only. Remove the protection
300 // temporarily.
301 if (IsPkeyPoolInitialized()) {
302 TagGlobalsWithPkey(kDefaultPkey);
303 }
304 #endif
305
306 PA_CHECK(pool_base);
307 PA_CHECK(size <= kConfigurablePoolMaxSize);
308 PA_CHECK(size >= kConfigurablePoolMinSize);
309 PA_CHECK(base::bits::IsPowerOfTwo(size));
310 PA_CHECK(pool_base % size == 0);
311
312 setup_.configurable_pool_base_address_ = pool_base;
313 setup_.configurable_pool_base_mask_ = ~(size - 1);
314
315 AddressPoolManager::GetInstance().Add(
316 kConfigurablePoolHandle, setup_.configurable_pool_base_address_, size);
317
318 #if BUILDFLAG(ENABLE_PKEYS)
319 // Put the pkey protection back in place.
320 if (IsPkeyPoolInitialized()) {
321 TagGlobalsWithPkey(setup_.pkey_);
322 }
323 #endif
324 }
325
326 #if BUILDFLAG(ENABLE_PKEYS)
InitPkeyPool(int pkey)327 void PartitionAddressSpace::InitPkeyPool(int pkey) {
328 // The PkeyPool can't be initialized with conflicting pkeys.
329 if (IsPkeyPoolInitialized()) {
330 PA_CHECK(setup_.pkey_ == pkey);
331 return;
332 }
333
334 size_t pool_size = PkeyPoolSize();
335 setup_.pkey_pool_base_address_ =
336 AllocPages(pool_size, pool_size,
337 PageAccessibilityConfiguration(
338 PageAccessibilityConfiguration::kInaccessible),
339 PageTag::kPartitionAlloc);
340 if (!setup_.pkey_pool_base_address_) {
341 HandlePoolAllocFailure();
342 }
343
344 PA_DCHECK(!(setup_.pkey_pool_base_address_ & (pool_size - 1)));
345 setup_.pkey_ = pkey;
346 AddressPoolManager::GetInstance().Add(
347 kPkeyPoolHandle, setup_.pkey_pool_base_address_, pool_size);
348
349 PA_DCHECK(!IsInPkeyPool(setup_.pkey_pool_base_address_ - 1));
350 PA_DCHECK(IsInPkeyPool(setup_.pkey_pool_base_address_));
351 PA_DCHECK(IsInPkeyPool(setup_.pkey_pool_base_address_ + pool_size - 1));
352 PA_DCHECK(!IsInPkeyPool(setup_.pkey_pool_base_address_ + pool_size));
353
354 // TODO(1362969): support PA_ENABLE_SHADOW_METADATA
355 }
356 #endif // BUILDFLAG(ENABLE_PKEYS)
357
UninitForTesting()358 void PartitionAddressSpace::UninitForTesting() {
359 #if BUILDFLAG(ENABLE_PKEYS)
360 UninitPkeyPoolForTesting(); // IN-TEST
361 #endif
362 #if PA_CONFIG(GLUE_CORE_POOLS)
363 // The core pools (regular & BRP) were allocated using a single allocation of
364 // double size.
365 FreePages(setup_.regular_pool_base_address_, 2 * RegularPoolSize());
366 #else // PA_CONFIG(GLUE_CORE_POOLS)
367 FreePages(setup_.regular_pool_base_address_, RegularPoolSize());
368 // For BRP pool, the allocation region includes a "forbidden zone" before the
369 // pool.
370 const size_t kForbiddenZoneSize = PageAllocationGranularity();
371 FreePages(setup_.brp_pool_base_address_ - kForbiddenZoneSize,
372 BRPPoolSize() + kForbiddenZoneSize);
373 #endif // PA_CONFIG(GLUE_CORE_POOLS)
374 // Do not free pages for the configurable pool, because its memory is owned
375 // by someone else, but deinitialize it nonetheless.
376 setup_.regular_pool_base_address_ = kUninitializedPoolBaseAddress;
377 setup_.brp_pool_base_address_ = kUninitializedPoolBaseAddress;
378 setup_.configurable_pool_base_address_ = kUninitializedPoolBaseAddress;
379 setup_.configurable_pool_base_mask_ = 0;
380 AddressPoolManager::GetInstance().ResetForTesting();
381 #if PA_CONFIG(POINTER_COMPRESSION)
382 CompressedPointerBaseGlobal::ResetBaseForTesting();
383 #endif // PA_CONFIG(POINTER_COMPRESSION)
384 }
385
UninitConfigurablePoolForTesting()386 void PartitionAddressSpace::UninitConfigurablePoolForTesting() {
387 #if BUILDFLAG(ENABLE_PKEYS)
388 // It's possible that the pkey pool has been initialized first, in which case
389 // the setup_ memory has been made read-only. Remove the protection
390 // temporarily.
391 if (IsPkeyPoolInitialized()) {
392 TagGlobalsWithPkey(kDefaultPkey);
393 }
394 #endif
395 AddressPoolManager::GetInstance().Remove(kConfigurablePoolHandle);
396 setup_.configurable_pool_base_address_ = kUninitializedPoolBaseAddress;
397 setup_.configurable_pool_base_mask_ = 0;
398 #if BUILDFLAG(ENABLE_PKEYS)
399 // Put the pkey protection back in place.
400 if (IsPkeyPoolInitialized()) {
401 TagGlobalsWithPkey(setup_.pkey_);
402 }
403 #endif
404 }
405
406 #if BUILDFLAG(ENABLE_PKEYS)
UninitPkeyPoolForTesting()407 void PartitionAddressSpace::UninitPkeyPoolForTesting() {
408 if (IsPkeyPoolInitialized()) {
409 TagGlobalsWithPkey(kDefaultPkey);
410 PkeySettings::settings.enabled = false;
411
412 FreePages(setup_.pkey_pool_base_address_, PkeyPoolSize());
413 AddressPoolManager::GetInstance().Remove(kPkeyPoolHandle);
414 setup_.pkey_pool_base_address_ = kUninitializedPoolBaseAddress;
415 setup_.pkey_ = kInvalidPkey;
416 }
417 }
418 #endif
419
420 #if BUILDFLAG(IS_LINUX) && defined(ARCH_CPU_ARM64)
421
422 PageCharacteristics page_characteristics;
423
424 #endif // BUILDFLAG(IS_LINUX) && defined(ARCH_CPU_ARM64)
425
426 #endif // BUILDFLAG(HAS_64_BIT_POINTERS)
427
428 } // namespace partition_alloc::internal
429