• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 // Copyright (c) 2005, Google Inc.
2 // All rights reserved.
3 //
4 // Redistribution and use in source and binary forms, with or without
5 // modification, are permitted provided that the following conditions are
6 // met:
7 //
8 //     * Redistributions of source code must retain the above copyright
9 // notice, this list of conditions and the following disclaimer.
10 //     * Redistributions in binary form must reproduce the above
11 // copyright notice, this list of conditions and the following disclaimer
12 // in the documentation and/or other materials provided with the
13 // distribution.
14 //     * Neither the name of Google Inc. nor the names of its
15 // contributors may be used to endorse or promote products derived from
16 // this software without specific prior written permission.
17 //
18 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 
30 // ---
31 // Author: Sanjay Ghemawat
32 
33 #include <config.h>
34 #include <errno.h>                      // for EAGAIN, errno
35 #include <fcntl.h>                      // for open, O_RDWR
36 #include <stddef.h>                     // for size_t, NULL, ptrdiff_t
37 #if defined HAVE_STDINT_H
38 #include <stdint.h>                     // for uintptr_t, intptr_t
39 #elif defined HAVE_INTTYPES_H
40 #include <inttypes.h>
41 #else
42 #include <sys/types.h>
43 #endif
44 #ifdef HAVE_MMAP
45 #include <sys/mman.h>                   // for munmap, mmap, MADV_DONTNEED, etc
46 #endif
47 #ifdef HAVE_UNISTD_H
48 #include <unistd.h>                     // for sbrk, getpagesize, off_t
49 #endif
50 #include <new>                          // for operator new
51 #include <gperftools/malloc_extension.h>
52 #include "base/basictypes.h"
53 #include "base/commandlineflags.h"
54 #include "base/spinlock.h"              // for SpinLockHolder, SpinLock, etc
55 #include "common.h"
56 #include "internal_logging.h"
57 
58 // On systems (like freebsd) that don't define MAP_ANONYMOUS, use the old
59 // form of the name instead.
60 #ifndef MAP_ANONYMOUS
61 # define MAP_ANONYMOUS MAP_ANON
62 #endif
63 
64 // MADV_FREE is specifically designed for use by malloc(), but only
65 // FreeBSD supports it; in linux we fall back to the somewhat inferior
66 // MADV_DONTNEED.
67 #if !defined(MADV_FREE) && defined(MADV_DONTNEED)
68 # define MADV_FREE  MADV_DONTNEED
69 #endif
70 
71 // Solaris has a bug where it doesn't declare madvise() for C++.
72 //    http://www.opensolaris.org/jive/thread.jspa?threadID=21035&tstart=0
73 #if defined(__sun) && defined(__SVR4)
74 # include <sys/types.h>    // for caddr_t
75   extern "C" { extern int madvise(caddr_t, size_t, int); }
76 #endif
77 
78 // Set kDebugMode mode so that we can have use C++ conditionals
79 // instead of preprocessor conditionals.
80 #ifdef NDEBUG
81 static const bool kDebugMode = false;
82 #else
83 static const bool kDebugMode = true;
84 #endif
85 
86 // TODO(sanjay): Move the code below into the tcmalloc namespace
87 using tcmalloc::kLog;
88 using tcmalloc::Log;
89 
90 // Anonymous namespace to avoid name conflicts on "CheckAddressBits".
91 namespace {
92 
93 // Check that no bit is set at position ADDRESS_BITS or higher.
CheckAddressBits(uintptr_t ptr)94 template <int ADDRESS_BITS> bool CheckAddressBits(uintptr_t ptr) {
95   return (ptr >> ADDRESS_BITS) == 0;
96 }
97 
98 // Specialize for the bit width of a pointer to avoid undefined shift.
CheckAddressBits(uintptr_t ptr)99 template <> bool CheckAddressBits<8 * sizeof(void*)>(uintptr_t ptr) {
100   return true;
101 }
102 
103 #if defined(OS_LINUX) && defined(__x86_64__)
104 #define ASLR_IS_SUPPORTED
105 #endif
106 
107 #if defined(ASLR_IS_SUPPORTED)
108 // From libdieharder, public domain library by Bob Jenkins (rngav.c).
109 // Described at http://burtleburtle.net/bob/rand/smallprng.html.
110 // Not cryptographically secure, but good enough for what we need.
111 typedef uint32_t u4;
112 struct ranctx {
113   u4 a;
114   u4 b;
115   u4 c;
116   u4 d;
117 };
118 
119 #define rot(x,k) (((x)<<(k))|((x)>>(32-(k))))
120 
ranval(ranctx * x)121 u4 ranval(ranctx* x) {
122   /* xxx: the generator being tested */
123   u4 e = x->a - rot(x->b, 27);
124   x->a = x->b ^ rot(x->c, 17);
125   x->b = x->c + x->d;
126   x->c = x->d + e;
127   x->d = e + x->a;
128   return x->d;
129 }
130 
raninit(ranctx * x,u4 seed)131 void raninit(ranctx* x, u4 seed) {
132   u4 i;
133   x->a = 0xf1ea5eed;
134   x->b = x->c = x->d = seed;
135   for (i = 0; i < 20; ++i) {
136     (void) ranval(x);
137   }
138 }
139 
140 // If the kernel cannot honor the hint in arch_get_unmapped_area_topdown, it
141 // will simply ignore it. So we give a hint that has a good chance of
142 // working.
143 // The mmap top-down allocator will normally allocate below TASK_SIZE - gap,
144 // with a gap that depends on the max stack size. See x86/mm/mmap.c. We
145 // should make allocations that are below this area, which would be
146 // 0x7ffbf8000000.
147 // We use 0x3ffffffff000 as the mask so that we only "pollute" half of the
148 // address space. In the unlikely case where fragmentation would become an
149 // issue, the kernel will still have another half to use.
150 const uint64_t kRandomAddressMask = 0x3ffffffff000ULL;
151 
152 #endif  // defined(ASLR_IS_SUPPORTED)
153 
154 // Give a random "hint" that is suitable for use with mmap(). This cannot make
155 // mmap fail, as the kernel will simply not follow the hint if it can't.
156 // However, this will create address space fragmentation.  Currently, we only
157 // implement it on x86_64, where we have a 47 bits userland address space and
158 // fragmentation is not an issue.
GetRandomAddrHint()159 void* GetRandomAddrHint() {
160 #if !defined(ASLR_IS_SUPPORTED)
161   return NULL;
162 #else
163   // Note: we are protected by the general TCMalloc_SystemAlloc spinlock. Given
164   // the nature of what we're doing, it wouldn't be critical if we weren't for
165   // ctx, but it is for the "initialized" variable.
166   // It's nice to share the state between threads, because scheduling will add
167   // some randomness to the succession of ranval() calls.
168   static ranctx ctx;
169   static bool initialized = false;
170   if (!initialized) {
171     initialized = true;
172     // We really want this to be a stack variable and don't want any compiler
173     // optimization. We're using its address as a poor-man source of
174     // randomness.
175     volatile char c;
176     // Pre-initialize our seed with a "random" address in case /dev/urandom is
177     // not available.
178     uint32_t seed = (reinterpret_cast<uint64_t>(&c) >> 32) ^
179                     reinterpret_cast<uint64_t>(&c);
180     int urandom_fd = open("/dev/urandom", O_RDONLY);
181     if (urandom_fd >= 0) {
182       ssize_t len;
183       len = read(urandom_fd, &seed, sizeof(seed));
184       ASSERT(len == sizeof(seed));
185       int ret = close(urandom_fd);
186       ASSERT(ret == 0);
187     }
188     raninit(&ctx, seed);
189   }
190   uint64_t random_address = (static_cast<uint64_t>(ranval(&ctx)) << 32) |
191                             ranval(&ctx);
192   // A a bit-wise "and" won't bias our random distribution.
193   random_address &= kRandomAddressMask;
194   return reinterpret_cast<void*>(random_address);
195 #endif  // ASLR_IS_SUPPORTED
196 }
197 
198 // Allocate |length| bytes of memory using mmap(). The memory will be
199 // readable and writeable, but not executable.
200 // Like mmap(), we will return MAP_FAILED on failure.
201 // |is_aslr_enabled| controls address space layout randomization. When true, we
202 // will put the first mapping at a random address and will then try to grow it.
203 // If it's not possible to grow an existing mapping, a new one will be created.
AllocWithMmap(size_t length,bool is_aslr_enabled)204 void* AllocWithMmap(size_t length, bool is_aslr_enabled) {
205   // Note: we are protected by the general TCMalloc_SystemAlloc spinlock.
206   static void* address_hint = NULL;
207 #if defined(ASLR_IS_SUPPORTED)
208   if (is_aslr_enabled &&
209       (!address_hint ||
210        reinterpret_cast<uint64_t>(address_hint) & ~kRandomAddressMask)) {
211     address_hint = GetRandomAddrHint();
212   }
213 #endif  // ASLR_IS_SUPPORTED
214 
215   // address_hint is likely to make us grow an existing mapping.
216   void* result = mmap(address_hint, length, PROT_READ|PROT_WRITE,
217                       MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
218 #if defined(ASLR_IS_SUPPORTED)
219   if (result == address_hint) {
220     // If mmap() succeeded at a address_hint, our next mmap() will try to grow
221     // the current mapping as long as it's compatible with our ASLR mask.
222     // This has been done for performance reasons, see crbug.com/173371.
223     // It should be possible to strike a better balance between performance
224     // and security but will be done at a later date.
225     // If this overflows, it could only set address_hint to NULL, which is
226     // what we want (and can't happen on the currently supported architecture).
227     address_hint = static_cast<char*>(result) + length;
228   } else {
229     // mmap failed or a collision prevented the kernel from honoring the hint,
230     // reset the hint.
231     address_hint = NULL;
232   }
233 #endif  // ASLR_IS_SUPPORTED
234   return result;
235 }
236 
237 }  // Anonymous namespace to avoid name conflicts on "CheckAddressBits".
238 
239 COMPILE_ASSERT(kAddressBits <= 8 * sizeof(void*),
240                address_bits_larger_than_pointer_size);
241 
242 // Structure for discovering alignment
243 union MemoryAligner {
244   void*  p;
245   double d;
246   size_t s;
247 } CACHELINE_ALIGNED;
248 
249 static SpinLock spinlock(SpinLock::LINKER_INITIALIZED);
250 
251 #if defined(HAVE_MMAP) || defined(MADV_FREE)
252 #ifdef HAVE_GETPAGESIZE
253 static size_t pagesize = 0;
254 #endif
255 #endif
256 
257 // The current system allocator
258 SysAllocator* sys_alloc = NULL;
259 
260 // Configuration parameters.
261 DEFINE_int32(malloc_devmem_start,
262              EnvToInt("TCMALLOC_DEVMEM_START", 0),
263              "Physical memory starting location in MB for /dev/mem allocation."
264              "  Setting this to 0 disables /dev/mem allocation");
265 DEFINE_int32(malloc_devmem_limit,
266              EnvToInt("TCMALLOC_DEVMEM_LIMIT", 0),
267              "Physical memory limit location in MB for /dev/mem allocation."
268              "  Setting this to 0 means no limit.");
269 DEFINE_bool(malloc_skip_sbrk,
270             EnvToBool("TCMALLOC_SKIP_SBRK", false),
271             "Whether sbrk can be used to obtain memory.");
272 DEFINE_bool(malloc_skip_mmap,
273             EnvToBool("TCMALLOC_SKIP_MMAP", false),
274             "Whether mmap can be used to obtain memory.");
275 
276 DEFINE_bool(malloc_random_allocator,
277 #if defined(ASLR_IS_SUPPORTED)
278             EnvToBool("TCMALLOC_ASLR", true),
279 #else
280             EnvToBool("TCMALLOC_ASLR", false),
281 #endif
282             "Whether to randomize the address space via mmap().");
283 
284 // static allocators
285 class SbrkSysAllocator : public SysAllocator {
286 public:
SbrkSysAllocator()287   SbrkSysAllocator() : SysAllocator() {
288   }
289   void* Alloc(size_t size, size_t *actual_size, size_t alignment);
290 };
291 static char sbrk_space[sizeof(SbrkSysAllocator)];
292 
293 class MmapSysAllocator : public SysAllocator {
294 public:
MmapSysAllocator()295   MmapSysAllocator() : SysAllocator() {
296   }
297   void* Alloc(size_t size, size_t *actual_size, size_t alignment);
298 };
299 static char mmap_space[sizeof(MmapSysAllocator)];
300 
301 class DevMemSysAllocator : public SysAllocator {
302 public:
DevMemSysAllocator()303   DevMemSysAllocator() : SysAllocator() {
304   }
305   void* Alloc(size_t size, size_t *actual_size, size_t alignment);
306 };
307 
308 class DefaultSysAllocator : public SysAllocator {
309  public:
DefaultSysAllocator()310   DefaultSysAllocator() : SysAllocator() {
311     for (int i = 0; i < kMaxAllocators; i++) {
312       failed_[i] = true;
313       allocs_[i] = NULL;
314       names_[i] = NULL;
315     }
316   }
SetChildAllocator(SysAllocator * alloc,unsigned int index,const char * name)317   void SetChildAllocator(SysAllocator* alloc, unsigned int index,
318                          const char* name) {
319     if (index < kMaxAllocators && alloc != NULL) {
320       allocs_[index] = alloc;
321       failed_[index] = false;
322       names_[index] = name;
323     }
324   }
325   void* Alloc(size_t size, size_t *actual_size, size_t alignment);
326 
327  private:
328   static const int kMaxAllocators = 2;
329   bool failed_[kMaxAllocators];
330   SysAllocator* allocs_[kMaxAllocators];
331   const char* names_[kMaxAllocators];
332 };
333 static char default_space[sizeof(DefaultSysAllocator)];
334 static const char sbrk_name[] = "SbrkSysAllocator";
335 static const char mmap_name[] = "MmapSysAllocator";
336 
337 
Alloc(size_t size,size_t * actual_size,size_t alignment)338 void* SbrkSysAllocator::Alloc(size_t size, size_t *actual_size,
339                               size_t alignment) {
340 #ifndef HAVE_SBRK
341   return NULL;
342 #else
343   // Check if we should use sbrk allocation.
344   // FLAGS_malloc_skip_sbrk starts out as false (its uninitialized
345   // state) and eventually gets initialized to the specified value.  Note
346   // that this code runs for a while before the flags are initialized.
347   // That means that even if this flag is set to true, some (initial)
348   // memory will be allocated with sbrk before the flag takes effect.
349   if (FLAGS_malloc_skip_sbrk) {
350     return NULL;
351   }
352 
353   // sbrk will release memory if passed a negative number, so we do
354   // a strict check here
355   if (static_cast<ptrdiff_t>(size + alignment) < 0) return NULL;
356 
357   // This doesn't overflow because TCMalloc_SystemAlloc has already
358   // tested for overflow at the alignment boundary.
359   size = ((size + alignment - 1) / alignment) * alignment;
360 
361   // "actual_size" indicates that the bytes from the returned pointer
362   // p up to and including (p + actual_size - 1) have been allocated.
363   if (actual_size) {
364     *actual_size = size;
365   }
366 
367   // Check that we we're not asking for so much more memory that we'd
368   // wrap around the end of the virtual address space.  (This seems
369   // like something sbrk() should check for us, and indeed opensolaris
370   // does, but glibc does not:
371   //    http://src.opensolaris.org/source/xref/onnv/onnv-gate/usr/src/lib/libc/port/sys/sbrk.c?a=true
372   //    http://sourceware.org/cgi-bin/cvsweb.cgi/~checkout~/libc/misc/sbrk.c?rev=1.1.2.1&content-type=text/plain&cvsroot=glibc
373   // Without this check, sbrk may succeed when it ought to fail.)
374   if (reinterpret_cast<intptr_t>(sbrk(0)) + size < size) {
375     return NULL;
376   }
377 
378   void* result = sbrk(size);
379   if (result == reinterpret_cast<void*>(-1)) {
380     return NULL;
381   }
382 
383   // Is it aligned?
384   uintptr_t ptr = reinterpret_cast<uintptr_t>(result);
385   if ((ptr & (alignment-1)) == 0)  return result;
386 
387   // Try to get more memory for alignment
388   size_t extra = alignment - (ptr & (alignment-1));
389   void* r2 = sbrk(extra);
390   if (reinterpret_cast<uintptr_t>(r2) == (ptr + size)) {
391     // Contiguous with previous result
392     return reinterpret_cast<void*>(ptr + extra);
393   }
394 
395   // Give up and ask for "size + alignment - 1" bytes so
396   // that we can find an aligned region within it.
397   result = sbrk(size + alignment - 1);
398   if (result == reinterpret_cast<void*>(-1)) {
399     return NULL;
400   }
401   ptr = reinterpret_cast<uintptr_t>(result);
402   if ((ptr & (alignment-1)) != 0) {
403     ptr += alignment - (ptr & (alignment-1));
404   }
405   return reinterpret_cast<void*>(ptr);
406 #endif  // HAVE_SBRK
407 }
408 
Alloc(size_t size,size_t * actual_size,size_t alignment)409 void* MmapSysAllocator::Alloc(size_t size, size_t *actual_size,
410                               size_t alignment) {
411 #ifndef HAVE_MMAP
412   return NULL;
413 #else
414   // Check if we should use mmap allocation.
415   // FLAGS_malloc_skip_mmap starts out as false (its uninitialized
416   // state) and eventually gets initialized to the specified value.  Note
417   // that this code runs for a while before the flags are initialized.
418   // Chances are we never get here before the flags are initialized since
419   // sbrk is used until the heap is exhausted (before mmap is used).
420   if (FLAGS_malloc_skip_mmap) {
421     return NULL;
422   }
423 
424   // Enforce page alignment
425   if (pagesize == 0) pagesize = getpagesize();
426   if (alignment < pagesize) alignment = pagesize;
427   size_t aligned_size = ((size + alignment - 1) / alignment) * alignment;
428   if (aligned_size < size) {
429     return NULL;
430   }
431   size = aligned_size;
432 
433   // "actual_size" indicates that the bytes from the returned pointer
434   // p up to and including (p + actual_size - 1) have been allocated.
435   if (actual_size) {
436     *actual_size = size;
437   }
438 
439   // Ask for extra memory if alignment > pagesize
440   size_t extra = 0;
441   if (alignment > pagesize) {
442     extra = alignment - pagesize;
443   }
444 
445   // Note: size + extra does not overflow since:
446   //            size + alignment < (1<<NBITS).
447   // and        extra <= alignment
448   // therefore  size + extra < (1<<NBITS)
449   void* result = AllocWithMmap(size + extra, FLAGS_malloc_random_allocator);
450   if (result == reinterpret_cast<void*>(MAP_FAILED)) {
451     return NULL;
452   }
453 
454   // Adjust the return memory so it is aligned
455   uintptr_t ptr = reinterpret_cast<uintptr_t>(result);
456   size_t adjust = 0;
457   if ((ptr & (alignment - 1)) != 0) {
458     adjust = alignment - (ptr & (alignment - 1));
459   }
460 
461   // Return the unused memory to the system
462   if (adjust > 0) {
463     munmap(reinterpret_cast<void*>(ptr), adjust);
464   }
465   if (adjust < extra) {
466     munmap(reinterpret_cast<void*>(ptr + adjust + size), extra - adjust);
467   }
468 
469   ptr += adjust;
470   return reinterpret_cast<void*>(ptr);
471 #endif  // HAVE_MMAP
472 }
473 
Alloc(size_t size,size_t * actual_size,size_t alignment)474 void* DevMemSysAllocator::Alloc(size_t size, size_t *actual_size,
475                                 size_t alignment) {
476 #ifndef HAVE_MMAP
477   return NULL;
478 #else
479   static bool initialized = false;
480   static off_t physmem_base;  // next physical memory address to allocate
481   static off_t physmem_limit; // maximum physical address allowed
482   static int physmem_fd;      // file descriptor for /dev/mem
483 
484   // Check if we should use /dev/mem allocation.  Note that it may take
485   // a while to get this flag initialized, so meanwhile we fall back to
486   // the next allocator.  (It looks like 7MB gets allocated before
487   // this flag gets initialized -khr.)
488   if (FLAGS_malloc_devmem_start == 0) {
489     // NOTE: not a devmem_failure - we'd like TCMalloc_SystemAlloc to
490     // try us again next time.
491     return NULL;
492   }
493 
494   if (!initialized) {
495     physmem_fd = open("/dev/mem", O_RDWR);
496     if (physmem_fd < 0) {
497       return NULL;
498     }
499     physmem_base = FLAGS_malloc_devmem_start*1024LL*1024LL;
500     physmem_limit = FLAGS_malloc_devmem_limit*1024LL*1024LL;
501     initialized = true;
502   }
503 
504   // Enforce page alignment
505   if (pagesize == 0) pagesize = getpagesize();
506   if (alignment < pagesize) alignment = pagesize;
507   size_t aligned_size = ((size + alignment - 1) / alignment) * alignment;
508   if (aligned_size < size) {
509     return NULL;
510   }
511   size = aligned_size;
512 
513   // "actual_size" indicates that the bytes from the returned pointer
514   // p up to and including (p + actual_size - 1) have been allocated.
515   if (actual_size) {
516     *actual_size = size;
517   }
518 
519   // Ask for extra memory if alignment > pagesize
520   size_t extra = 0;
521   if (alignment > pagesize) {
522     extra = alignment - pagesize;
523   }
524 
525   // check to see if we have any memory left
526   if (physmem_limit != 0 &&
527       ((size + extra) > (physmem_limit - physmem_base))) {
528     return NULL;
529   }
530 
531   // Note: size + extra does not overflow since:
532   //            size + alignment < (1<<NBITS).
533   // and        extra <= alignment
534   // therefore  size + extra < (1<<NBITS)
535   void *result = mmap(0, size + extra, PROT_WRITE|PROT_READ,
536                       MAP_SHARED, physmem_fd, physmem_base);
537   if (result == reinterpret_cast<void*>(MAP_FAILED)) {
538     return NULL;
539   }
540   uintptr_t ptr = reinterpret_cast<uintptr_t>(result);
541 
542   // Adjust the return memory so it is aligned
543   size_t adjust = 0;
544   if ((ptr & (alignment - 1)) != 0) {
545     adjust = alignment - (ptr & (alignment - 1));
546   }
547 
548   // Return the unused virtual memory to the system
549   if (adjust > 0) {
550     munmap(reinterpret_cast<void*>(ptr), adjust);
551   }
552   if (adjust < extra) {
553     munmap(reinterpret_cast<void*>(ptr + adjust + size), extra - adjust);
554   }
555 
556   ptr += adjust;
557   physmem_base += adjust + size;
558 
559   return reinterpret_cast<void*>(ptr);
560 #endif  // HAVE_MMAP
561 }
562 
Alloc(size_t size,size_t * actual_size,size_t alignment)563 void* DefaultSysAllocator::Alloc(size_t size, size_t *actual_size,
564                                  size_t alignment) {
565   for (int i = 0; i < kMaxAllocators; i++) {
566     if (!failed_[i] && allocs_[i] != NULL) {
567       void* result = allocs_[i]->Alloc(size, actual_size, alignment);
568       if (result != NULL) {
569         return result;
570       }
571       failed_[i] = true;
572     }
573   }
574   // After both failed, reset "failed_" to false so that a single failed
575   // allocation won't make the allocator never work again.
576   for (int i = 0; i < kMaxAllocators; i++) {
577     failed_[i] = false;
578   }
579   return NULL;
580 }
581 
582 static bool system_alloc_inited = false;
InitSystemAllocators(void)583 void InitSystemAllocators(void) {
584   MmapSysAllocator *mmap = new (mmap_space) MmapSysAllocator();
585   SbrkSysAllocator *sbrk = new (sbrk_space) SbrkSysAllocator();
586 
587   // In 64-bit debug mode, place the mmap allocator first since it
588   // allocates pointers that do not fit in 32 bits and therefore gives
589   // us better testing of code's 64-bit correctness.  It also leads to
590   // less false negatives in heap-checking code.  (Numbers are less
591   // likely to look like pointers and therefore the conservative gc in
592   // the heap-checker is less likely to misinterpret a number as a
593   // pointer).
594   DefaultSysAllocator *sdef = new (default_space) DefaultSysAllocator();
595   // Unfortunately, this code runs before flags are initialized. So
596   // we can't use FLAGS_malloc_random_allocator.
597 #if defined(ASLR_IS_SUPPORTED)
598   // Our only random allocator is mmap.
599   sdef->SetChildAllocator(mmap, 0, mmap_name);
600 #else
601   if (kDebugMode && sizeof(void*) > 4) {
602     sdef->SetChildAllocator(mmap, 0, mmap_name);
603     sdef->SetChildAllocator(sbrk, 1, sbrk_name);
604   } else {
605     sdef->SetChildAllocator(sbrk, 0, sbrk_name);
606     sdef->SetChildAllocator(mmap, 1, mmap_name);
607   }
608 #endif  // ASLR_IS_SUPPORTED
609   sys_alloc = sdef;
610 }
611 
TCMalloc_SystemAlloc(size_t size,size_t * actual_size,size_t alignment)612 void* TCMalloc_SystemAlloc(size_t size, size_t *actual_size,
613                            size_t alignment) {
614   // Discard requests that overflow
615   if (size + alignment < size) return NULL;
616 
617   SpinLockHolder lock_holder(&spinlock);
618 
619   if (!system_alloc_inited) {
620     InitSystemAllocators();
621     system_alloc_inited = true;
622   }
623 
624   // Enforce minimum alignment
625   if (alignment < sizeof(MemoryAligner)) alignment = sizeof(MemoryAligner);
626 
627   void* result = sys_alloc->Alloc(size, actual_size, alignment);
628   if (result != NULL) {
629     if (actual_size) {
630       CheckAddressBits<kAddressBits>(
631           reinterpret_cast<uintptr_t>(result) + *actual_size - 1);
632     } else {
633       CheckAddressBits<kAddressBits>(
634           reinterpret_cast<uintptr_t>(result) + size - 1);
635     }
636   }
637   return result;
638 }
639 
TCMalloc_SystemAddGuard(void * start,size_t size)640 size_t TCMalloc_SystemAddGuard(void* start, size_t size) {
641 #ifdef HAVE_GETPAGESIZE
642   if (pagesize == 0)
643     pagesize = getpagesize();
644 
645   if (size < pagesize || (reinterpret_cast<size_t>(start) % pagesize) != 0)
646     return 0;
647 
648   if (!mprotect(start, pagesize, PROT_NONE))
649     return pagesize;
650 #endif
651 
652   return 0;
653 }
654 
TCMalloc_SystemRelease(void * start,size_t length)655 void TCMalloc_SystemRelease(void* start, size_t length) {
656 #ifdef MADV_FREE
657   if (FLAGS_malloc_devmem_start) {
658     // It's not safe to use MADV_FREE/MADV_DONTNEED if we've been
659     // mapping /dev/mem for heap memory.
660     return;
661   }
662   if (pagesize == 0) pagesize = getpagesize();
663   const size_t pagemask = pagesize - 1;
664 
665   size_t new_start = reinterpret_cast<size_t>(start);
666   size_t end = new_start + length;
667   size_t new_end = end;
668 
669   // Round up the starting address and round down the ending address
670   // to be page aligned:
671   new_start = (new_start + pagesize - 1) & ~pagemask;
672   new_end = new_end & ~pagemask;
673 
674   ASSERT((new_start & pagemask) == 0);
675   ASSERT((new_end & pagemask) == 0);
676   ASSERT(new_start >= reinterpret_cast<size_t>(start));
677   ASSERT(new_end <= end);
678 
679   if (new_end > new_start) {
680     // Note -- ignoring most return codes, because if this fails it
681     // doesn't matter...
682     while (madvise(reinterpret_cast<char*>(new_start), new_end - new_start,
683                    MADV_FREE) == -1 &&
684            errno == EAGAIN) {
685       // NOP
686     }
687   }
688 #endif
689 }
690 
TCMalloc_SystemCommit(void * start,size_t length)691 void TCMalloc_SystemCommit(void* start, size_t length) {
692   // Nothing to do here.  TCMalloc_SystemRelease does not alter pages
693   // such that they need to be re-committed before they can be used by the
694   // application.
695 }
696