• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 //===-- sanitizer_coverage.cc ---------------------------------------------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // Sanitizer Coverage.
11 // This file implements run-time support for a poor man's coverage tool.
12 //
13 // Compiler instrumentation:
14 // For every interesting basic block the compiler injects the following code:
15 // if (Guard < 0) {
16 //    __sanitizer_cov(&Guard);
17 // }
18 // At the module start up time __sanitizer_cov_module_init sets the guards
19 // to consecutive negative numbers (-1, -2, -3, ...).
20 // It's fine to call __sanitizer_cov more than once for a given block.
21 //
22 // Run-time:
23 //  - __sanitizer_cov(): record that we've executed the PC (GET_CALLER_PC).
24 //    and atomically set Guard to -Guard.
25 //  - __sanitizer_cov_dump: dump the coverage data to disk.
26 //  For every module of the current process that has coverage data
27 //  this will create a file module_name.PID.sancov.
28 //
29 // The file format is simple: the first 8 bytes is the magic,
30 // one of 0xC0BFFFFFFFFFFF64 and 0xC0BFFFFFFFFFFF32. The last byte of the
31 // magic defines the size of the following offsets.
32 // The rest of the data is the offsets in the module.
33 //
34 // Eventually, this coverage implementation should be obsoleted by a more
35 // powerful general purpose Clang/LLVM coverage instrumentation.
36 // Consider this implementation as prototype.
37 //
38 // FIXME: support (or at least test with) dlclose.
39 //===----------------------------------------------------------------------===//
40 
41 #include "sanitizer_allocator_internal.h"
42 #include "sanitizer_common.h"
43 #include "sanitizer_libc.h"
44 #include "sanitizer_mutex.h"
45 #include "sanitizer_procmaps.h"
46 #include "sanitizer_stacktrace.h"
47 #include "sanitizer_symbolizer.h"
48 #include "sanitizer_flags.h"
49 
50 static const u64 kMagic64 = 0xC0BFFFFFFFFFFF64ULL;
51 static const u64 kMagic32 = 0xC0BFFFFFFFFFFF32ULL;
52 static const uptr kNumWordsForMagic = SANITIZER_WORDSIZE == 64 ? 1 : 2;
53 static const u64 kMagic = SANITIZER_WORDSIZE == 64 ? kMagic64 : kMagic32;
54 
55 static atomic_uint32_t dump_once_guard;  // Ensure that CovDump runs only once.
56 
57 static atomic_uintptr_t coverage_counter;
58 static atomic_uintptr_t caller_callee_counter;
59 
ResetGlobalCounters()60 static void ResetGlobalCounters() {
61   return atomic_store(&coverage_counter, 0, memory_order_relaxed);
62   return atomic_store(&caller_callee_counter, 0, memory_order_relaxed);
63 }
64 
65 // pc_array is the array containing the covered PCs.
66 // To make the pc_array thread- and async-signal-safe it has to be large enough.
67 // 128M counters "ought to be enough for anybody" (4M on 32-bit).
68 
69 // With coverage_direct=1 in ASAN_OPTIONS, pc_array memory is mapped to a file.
70 // In this mode, __sanitizer_cov_dump does nothing, and CovUpdateMapping()
71 // dump current memory layout to another file.
72 
73 static bool cov_sandboxed = false;
74 static fd_t cov_fd = kInvalidFd;
75 static unsigned int cov_max_block_size = 0;
76 static bool coverage_enabled = false;
77 static const char *coverage_dir;
78 
79 namespace __sanitizer {
80 
81 class CoverageData {
82  public:
83   void Init();
84   void Enable();
85   void Disable();
86   void ReInit();
87   void BeforeFork();
88   void AfterFork(int child_pid);
89   void Extend(uptr npcs);
90   void Add(uptr pc, u32 *guard);
91   void IndirCall(uptr caller, uptr callee, uptr callee_cache[],
92                  uptr cache_size);
93   void DumpCallerCalleePairs();
94   void DumpTrace();
95   void DumpAsBitSet();
96   void DumpCounters();
97   void DumpOffsets();
98   void DumpAll();
99 
100   ALWAYS_INLINE
101   void TraceBasicBlock(u32 *id);
102 
103   void InitializeGuardArray(s32 *guards);
104   void InitializeGuards(s32 *guards, uptr n, const char *module_name,
105                         uptr caller_pc);
106   void InitializeCounters(u8 *counters, uptr n);
107   void ReinitializeGuards();
108   uptr GetNumberOf8bitCounters();
109   uptr Update8bitCounterBitsetAndClearCounters(u8 *bitset);
110 
111   uptr *data();
112   uptr size() const;
buffer() const113   uptr *buffer() const { return pc_buffer; }
114 
115  private:
116   struct NamedPcRange {
117     const char *copied_module_name;
118     uptr beg, end; // elements [beg,end) in pc_array.
119   };
120 
121   void DirectOpen();
122   void UpdateModuleNameVec(uptr caller_pc, uptr range_beg, uptr range_end);
123   void GetRangeOffsets(const NamedPcRange& r, Symbolizer* s,
124       InternalMmapVector<uptr>* offsets) const;
125 
126   // Maximal size pc array may ever grow.
127   // We MmapNoReserve this space to ensure that the array is contiguous.
128   static const uptr kPcArrayMaxSize = FIRST_32_SECOND_64(
129       1 << (SANITIZER_ANDROID ? 24 : (SANITIZER_WINDOWS ? 27 : 26)),
130       1 << 27);
131   // The amount file mapping for the pc array is grown by.
132   static const uptr kPcArrayMmapSize = 64 * 1024;
133 
134   // pc_array is allocated with MmapNoReserveOrDie and so it uses only as
135   // much RAM as it really needs.
136   uptr *pc_array;
137   // Index of the first available pc_array slot.
138   atomic_uintptr_t pc_array_index;
139   // Array size.
140   atomic_uintptr_t pc_array_size;
141   // Current file mapped size of the pc array.
142   uptr pc_array_mapped_size;
143   // Descriptor of the file mapped pc array.
144   fd_t pc_fd;
145 
146   uptr *pc_buffer;
147 
148   // Vector of coverage guard arrays, protected by mu.
149   InternalMmapVectorNoCtor<s32*> guard_array_vec;
150 
151   // Vector of module and compilation unit pc ranges.
152   InternalMmapVectorNoCtor<NamedPcRange> comp_unit_name_vec;
153   InternalMmapVectorNoCtor<NamedPcRange> module_name_vec;
154 
155   struct CounterAndSize {
156     u8 *counters;
157     uptr n;
158   };
159 
160   InternalMmapVectorNoCtor<CounterAndSize> counters_vec;
161   uptr num_8bit_counters;
162 
163   // Caller-Callee (cc) array, size and current index.
164   static const uptr kCcArrayMaxSize = FIRST_32_SECOND_64(1 << 18, 1 << 24);
165   uptr **cc_array;
166   atomic_uintptr_t cc_array_index;
167   atomic_uintptr_t cc_array_size;
168 
169   // Tracing event array, size and current pointer.
170   // We record all events (basic block entries) in a global buffer of u32
171   // values. Each such value is the index in pc_array.
172   // So far the tracing is highly experimental:
173   //   - not thread-safe;
174   //   - does not support long traces;
175   //   - not tuned for performance.
176   static const uptr kTrEventArrayMaxSize = FIRST_32_SECOND_64(1 << 22, 1 << 30);
177   u32 *tr_event_array;
178   uptr tr_event_array_size;
179   u32 *tr_event_pointer;
180   static const uptr kTrPcArrayMaxSize    = FIRST_32_SECOND_64(1 << 22, 1 << 27);
181 
182   StaticSpinMutex mu;
183 };
184 
185 static CoverageData coverage_data;
186 
187 void CovUpdateMapping(const char *path, uptr caller_pc = 0);
188 
DirectOpen()189 void CoverageData::DirectOpen() {
190   InternalScopedString path(kMaxPathLength);
191   internal_snprintf((char *)path.data(), path.size(), "%s/%zd.sancov.raw",
192                     coverage_dir, internal_getpid());
193   pc_fd = OpenFile(path.data(), RdWr);
194   if (pc_fd == kInvalidFd) {
195     Report("Coverage: failed to open %s for reading/writing\n", path.data());
196     Die();
197   }
198 
199   pc_array_mapped_size = 0;
200   CovUpdateMapping(coverage_dir);
201 }
202 
Init()203 void CoverageData::Init() {
204   pc_fd = kInvalidFd;
205 }
206 
Enable()207 void CoverageData::Enable() {
208   if (pc_array)
209     return;
210   pc_array = reinterpret_cast<uptr *>(
211       MmapNoReserveOrDie(sizeof(uptr) * kPcArrayMaxSize, "CovInit"));
212   atomic_store(&pc_array_index, 0, memory_order_relaxed);
213   if (common_flags()->coverage_direct) {
214     atomic_store(&pc_array_size, 0, memory_order_relaxed);
215   } else {
216     atomic_store(&pc_array_size, kPcArrayMaxSize, memory_order_relaxed);
217   }
218 
219   pc_buffer = nullptr;
220   if (common_flags()->coverage_pc_buffer)
221     pc_buffer = reinterpret_cast<uptr *>(MmapNoReserveOrDie(
222         sizeof(uptr) * kPcArrayMaxSize, "CovInit::pc_buffer"));
223 
224   cc_array = reinterpret_cast<uptr **>(MmapNoReserveOrDie(
225       sizeof(uptr *) * kCcArrayMaxSize, "CovInit::cc_array"));
226   atomic_store(&cc_array_size, kCcArrayMaxSize, memory_order_relaxed);
227   atomic_store(&cc_array_index, 0, memory_order_relaxed);
228 
229   // Allocate tr_event_array with a guard page at the end.
230   tr_event_array = reinterpret_cast<u32 *>(MmapNoReserveOrDie(
231       sizeof(tr_event_array[0]) * kTrEventArrayMaxSize + GetMmapGranularity(),
232       "CovInit::tr_event_array"));
233   MprotectNoAccess(
234       reinterpret_cast<uptr>(&tr_event_array[kTrEventArrayMaxSize]),
235       GetMmapGranularity());
236   tr_event_array_size = kTrEventArrayMaxSize;
237   tr_event_pointer = tr_event_array;
238 
239   num_8bit_counters = 0;
240 }
241 
InitializeGuardArray(s32 * guards)242 void CoverageData::InitializeGuardArray(s32 *guards) {
243   Enable();  // Make sure coverage is enabled at this point.
244   s32 n = guards[0];
245   for (s32 j = 1; j <= n; j++) {
246     uptr idx = atomic_load_relaxed(&pc_array_index);
247     atomic_store_relaxed(&pc_array_index, idx + 1);
248     guards[j] = -static_cast<s32>(idx + 1);
249   }
250 }
251 
Disable()252 void CoverageData::Disable() {
253   if (pc_array) {
254     UnmapOrDie(pc_array, sizeof(uptr) * kPcArrayMaxSize);
255     pc_array = nullptr;
256   }
257   if (cc_array) {
258     UnmapOrDie(cc_array, sizeof(uptr *) * kCcArrayMaxSize);
259     cc_array = nullptr;
260   }
261   if (pc_buffer) {
262     UnmapOrDie(pc_buffer, sizeof(uptr) * kPcArrayMaxSize);
263     pc_buffer = nullptr;
264   }
265   if (tr_event_array) {
266     UnmapOrDie(tr_event_array,
267                sizeof(tr_event_array[0]) * kTrEventArrayMaxSize +
268                    GetMmapGranularity());
269     tr_event_array = nullptr;
270     tr_event_pointer = nullptr;
271   }
272   if (pc_fd != kInvalidFd) {
273     CloseFile(pc_fd);
274     pc_fd = kInvalidFd;
275   }
276 }
277 
ReinitializeGuards()278 void CoverageData::ReinitializeGuards() {
279   // Assuming single thread.
280   atomic_store(&pc_array_index, 0, memory_order_relaxed);
281   for (uptr i = 0; i < guard_array_vec.size(); i++)
282     InitializeGuardArray(guard_array_vec[i]);
283 }
284 
ReInit()285 void CoverageData::ReInit() {
286   Disable();
287   if (coverage_enabled) {
288     if (common_flags()->coverage_direct) {
289       // In memory-mapped mode we must extend the new file to the known array
290       // size.
291       uptr size = atomic_load(&pc_array_size, memory_order_relaxed);
292       uptr npcs = size / sizeof(uptr);
293       Enable();
294       if (size) Extend(npcs);
295       if (coverage_enabled) CovUpdateMapping(coverage_dir);
296     } else {
297       Enable();
298     }
299   }
300   // Re-initialize the guards.
301   // We are single-threaded now, no need to grab any lock.
302   CHECK_EQ(atomic_load(&pc_array_index, memory_order_relaxed), 0);
303   ReinitializeGuards();
304 }
305 
BeforeFork()306 void CoverageData::BeforeFork() {
307   mu.Lock();
308 }
309 
AfterFork(int child_pid)310 void CoverageData::AfterFork(int child_pid) {
311   // We are single-threaded so it's OK to release the lock early.
312   mu.Unlock();
313   if (child_pid == 0) ReInit();
314 }
315 
316 // Extend coverage PC array to fit additional npcs elements.
Extend(uptr npcs)317 void CoverageData::Extend(uptr npcs) {
318   if (!common_flags()->coverage_direct) return;
319   SpinMutexLock l(&mu);
320 
321   uptr size = atomic_load(&pc_array_size, memory_order_relaxed);
322   size += npcs * sizeof(uptr);
323 
324   if (coverage_enabled && size > pc_array_mapped_size) {
325     if (pc_fd == kInvalidFd) DirectOpen();
326     CHECK_NE(pc_fd, kInvalidFd);
327 
328     uptr new_mapped_size = pc_array_mapped_size;
329     while (size > new_mapped_size) new_mapped_size += kPcArrayMmapSize;
330     CHECK_LE(new_mapped_size, sizeof(uptr) * kPcArrayMaxSize);
331 
332     // Extend the file and map the new space at the end of pc_array.
333     uptr res = internal_ftruncate(pc_fd, new_mapped_size);
334     int err;
335     if (internal_iserror(res, &err)) {
336       Printf("failed to extend raw coverage file: %d\n", err);
337       Die();
338     }
339 
340     uptr next_map_base = ((uptr)pc_array) + pc_array_mapped_size;
341     void *p = MapWritableFileToMemory((void *)next_map_base,
342                                       new_mapped_size - pc_array_mapped_size,
343                                       pc_fd, pc_array_mapped_size);
344     CHECK_EQ((uptr)p, next_map_base);
345     pc_array_mapped_size = new_mapped_size;
346   }
347 
348   atomic_store(&pc_array_size, size, memory_order_release);
349 }
350 
InitializeCounters(u8 * counters,uptr n)351 void CoverageData::InitializeCounters(u8 *counters, uptr n) {
352   if (!counters) return;
353   CHECK_EQ(reinterpret_cast<uptr>(counters) % 16, 0);
354   n = RoundUpTo(n, 16); // The compiler must ensure that counters is 16-aligned.
355   SpinMutexLock l(&mu);
356   counters_vec.push_back({counters, n});
357   num_8bit_counters += n;
358 }
359 
UpdateModuleNameVec(uptr caller_pc,uptr range_beg,uptr range_end)360 void CoverageData::UpdateModuleNameVec(uptr caller_pc, uptr range_beg,
361                                        uptr range_end) {
362   auto sym = Symbolizer::GetOrInit();
363   if (!sym)
364     return;
365   const char *module_name = sym->GetModuleNameForPc(caller_pc);
366   if (!module_name) return;
367   if (module_name_vec.empty() ||
368       module_name_vec.back().copied_module_name != module_name)
369     module_name_vec.push_back({module_name, range_beg, range_end});
370   else
371     module_name_vec.back().end = range_end;
372 }
373 
InitializeGuards(s32 * guards,uptr n,const char * comp_unit_name,uptr caller_pc)374 void CoverageData::InitializeGuards(s32 *guards, uptr n,
375                                     const char *comp_unit_name,
376                                     uptr caller_pc) {
377   // The array 'guards' has n+1 elements, we use the element zero
378   // to store 'n'.
379   CHECK_LT(n, 1 << 30);
380   guards[0] = static_cast<s32>(n);
381   InitializeGuardArray(guards);
382   SpinMutexLock l(&mu);
383   uptr range_end = atomic_load(&pc_array_index, memory_order_relaxed);
384   uptr range_beg = range_end - n;
385   comp_unit_name_vec.push_back({comp_unit_name, range_beg, range_end});
386   guard_array_vec.push_back(guards);
387   UpdateModuleNameVec(caller_pc, range_beg, range_end);
388 }
389 
390 static const uptr kBundleCounterBits = 16;
391 
392 // When coverage_order_pcs==true and SANITIZER_WORDSIZE==64
393 // we insert the global counter into the first 16 bits of the PC.
BundlePcAndCounter(uptr pc,uptr counter)394 uptr BundlePcAndCounter(uptr pc, uptr counter) {
395   if (SANITIZER_WORDSIZE != 64 || !common_flags()->coverage_order_pcs)
396     return pc;
397   static const uptr kMaxCounter = (1 << kBundleCounterBits) - 1;
398   if (counter > kMaxCounter)
399     counter = kMaxCounter;
400   CHECK_EQ(0, pc >> (SANITIZER_WORDSIZE - kBundleCounterBits));
401   return pc | (counter << (SANITIZER_WORDSIZE - kBundleCounterBits));
402 }
403 
UnbundlePc(uptr bundle)404 uptr UnbundlePc(uptr bundle) {
405   if (SANITIZER_WORDSIZE != 64 || !common_flags()->coverage_order_pcs)
406     return bundle;
407   return (bundle << kBundleCounterBits) >> kBundleCounterBits;
408 }
409 
UnbundleCounter(uptr bundle)410 uptr UnbundleCounter(uptr bundle) {
411   if (SANITIZER_WORDSIZE != 64 || !common_flags()->coverage_order_pcs)
412     return 0;
413   return bundle >> (SANITIZER_WORDSIZE - kBundleCounterBits);
414 }
415 
416 // If guard is negative, atomically set it to -guard and store the PC in
417 // pc_array.
Add(uptr pc,u32 * guard)418 void CoverageData::Add(uptr pc, u32 *guard) {
419   atomic_uint32_t *atomic_guard = reinterpret_cast<atomic_uint32_t*>(guard);
420   s32 guard_value = atomic_load(atomic_guard, memory_order_relaxed);
421   if (guard_value >= 0) return;
422 
423   atomic_store(atomic_guard, -guard_value, memory_order_relaxed);
424   if (!pc_array) return;
425 
426   uptr idx = -guard_value - 1;
427   if (idx >= atomic_load(&pc_array_index, memory_order_acquire))
428     return;  // May happen after fork when pc_array_index becomes 0.
429   CHECK_LT(idx * sizeof(uptr),
430            atomic_load(&pc_array_size, memory_order_acquire));
431   uptr counter = atomic_fetch_add(&coverage_counter, 1, memory_order_relaxed);
432   pc_array[idx] = BundlePcAndCounter(pc, counter);
433   if (pc_buffer) pc_buffer[counter] = pc;
434 }
435 
436 // Registers a pair caller=>callee.
437 // When a given caller is seen for the first time, the callee_cache is added
438 // to the global array cc_array, callee_cache[0] is set to caller and
439 // callee_cache[1] is set to cache_size.
440 // Then we are trying to add callee to callee_cache [2,cache_size) if it is
441 // not there yet.
442 // If the cache is full we drop the callee (may want to fix this later).
IndirCall(uptr caller,uptr callee,uptr callee_cache[],uptr cache_size)443 void CoverageData::IndirCall(uptr caller, uptr callee, uptr callee_cache[],
444                              uptr cache_size) {
445   if (!cc_array) return;
446   atomic_uintptr_t *atomic_callee_cache =
447       reinterpret_cast<atomic_uintptr_t *>(callee_cache);
448   uptr zero = 0;
449   if (atomic_compare_exchange_strong(&atomic_callee_cache[0], &zero, caller,
450                                      memory_order_seq_cst)) {
451     uptr idx = atomic_fetch_add(&cc_array_index, 1, memory_order_relaxed);
452     CHECK_LT(idx * sizeof(uptr),
453              atomic_load(&cc_array_size, memory_order_acquire));
454     callee_cache[1] = cache_size;
455     cc_array[idx] = callee_cache;
456   }
457   CHECK_EQ(atomic_load(&atomic_callee_cache[0], memory_order_relaxed), caller);
458   for (uptr i = 2; i < cache_size; i++) {
459     uptr was = 0;
460     if (atomic_compare_exchange_strong(&atomic_callee_cache[i], &was, callee,
461                                        memory_order_seq_cst)) {
462       atomic_fetch_add(&caller_callee_counter, 1, memory_order_relaxed);
463       return;
464     }
465     if (was == callee)  // Already have this callee.
466       return;
467   }
468 }
469 
GetNumberOf8bitCounters()470 uptr CoverageData::GetNumberOf8bitCounters() {
471   return num_8bit_counters;
472 }
473 
474 // Map every 8bit counter to a 8-bit bitset and clear the counter.
Update8bitCounterBitsetAndClearCounters(u8 * bitset)475 uptr CoverageData::Update8bitCounterBitsetAndClearCounters(u8 *bitset) {
476   uptr num_new_bits = 0;
477   uptr cur = 0;
478   // For better speed we map 8 counters to 8 bytes of bitset at once.
479   static const uptr kBatchSize = 8;
480   CHECK_EQ(reinterpret_cast<uptr>(bitset) % kBatchSize, 0);
481   for (uptr i = 0, len = counters_vec.size(); i < len; i++) {
482     u8 *c = counters_vec[i].counters;
483     uptr n = counters_vec[i].n;
484     CHECK_EQ(n % 16, 0);
485     CHECK_EQ(cur % kBatchSize, 0);
486     CHECK_EQ(reinterpret_cast<uptr>(c) % kBatchSize, 0);
487     if (!bitset) {
488       internal_bzero_aligned16(c, n);
489       cur += n;
490       continue;
491     }
492     for (uptr j = 0; j < n; j += kBatchSize, cur += kBatchSize) {
493       CHECK_LT(cur, num_8bit_counters);
494       u64 *pc64 = reinterpret_cast<u64*>(c + j);
495       u64 *pb64 = reinterpret_cast<u64*>(bitset + cur);
496       u64 c64 = *pc64;
497       u64 old_bits_64 = *pb64;
498       u64 new_bits_64 = old_bits_64;
499       if (c64) {
500         *pc64 = 0;
501         for (uptr k = 0; k < kBatchSize; k++) {
502           u64 x = (c64 >> (8 * k)) & 0xff;
503           if (x) {
504             u64 bit = 0;
505             /**/ if (x >= 128) bit = 128;
506             else if (x >= 32) bit = 64;
507             else if (x >= 16) bit = 32;
508             else if (x >= 8) bit = 16;
509             else if (x >= 4) bit = 8;
510             else if (x >= 3) bit = 4;
511             else if (x >= 2) bit = 2;
512             else if (x >= 1) bit = 1;
513             u64 mask = bit << (8 * k);
514             if (!(new_bits_64 & mask)) {
515               num_new_bits++;
516               new_bits_64 |= mask;
517             }
518           }
519         }
520         *pb64 = new_bits_64;
521       }
522     }
523   }
524   CHECK_EQ(cur, num_8bit_counters);
525   return num_new_bits;
526 }
527 
data()528 uptr *CoverageData::data() {
529   return pc_array;
530 }
531 
size() const532 uptr CoverageData::size() const {
533   return atomic_load(&pc_array_index, memory_order_relaxed);
534 }
535 
536 // Block layout for packed file format: header, followed by module name (no
537 // trailing zero), followed by data blob.
538 struct CovHeader {
539   int pid;
540   unsigned int module_name_length;
541   unsigned int data_length;
542 };
543 
CovWritePacked(int pid,const char * module,const void * blob,unsigned int blob_size)544 static void CovWritePacked(int pid, const char *module, const void *blob,
545                            unsigned int blob_size) {
546   if (cov_fd == kInvalidFd) return;
547   unsigned module_name_length = internal_strlen(module);
548   CovHeader header = {pid, module_name_length, blob_size};
549 
550   if (cov_max_block_size == 0) {
551     // Writing to a file. Just go ahead.
552     WriteToFile(cov_fd, &header, sizeof(header));
553     WriteToFile(cov_fd, module, module_name_length);
554     WriteToFile(cov_fd, blob, blob_size);
555   } else {
556     // Writing to a socket. We want to split the data into appropriately sized
557     // blocks.
558     InternalScopedBuffer<char> block(cov_max_block_size);
559     CHECK_EQ((uptr)block.data(), (uptr)(CovHeader *)block.data());
560     uptr header_size_with_module = sizeof(header) + module_name_length;
561     CHECK_LT(header_size_with_module, cov_max_block_size);
562     unsigned int max_payload_size =
563         cov_max_block_size - header_size_with_module;
564     char *block_pos = block.data();
565     internal_memcpy(block_pos, &header, sizeof(header));
566     block_pos += sizeof(header);
567     internal_memcpy(block_pos, module, module_name_length);
568     block_pos += module_name_length;
569     char *block_data_begin = block_pos;
570     const char *blob_pos = (const char *)blob;
571     while (blob_size > 0) {
572       unsigned int payload_size = Min(blob_size, max_payload_size);
573       blob_size -= payload_size;
574       internal_memcpy(block_data_begin, blob_pos, payload_size);
575       blob_pos += payload_size;
576       ((CovHeader *)block.data())->data_length = payload_size;
577       WriteToFile(cov_fd, block.data(), header_size_with_module + payload_size);
578     }
579   }
580 }
581 
582 // If packed = false: <name>.<pid>.<sancov> (name = module name).
583 // If packed = true and name == 0: <pid>.<sancov>.<packed>.
584 // If packed = true and name != 0: <name>.<sancov>.<packed> (name is
585 // user-supplied).
CovOpenFile(InternalScopedString * path,bool packed,const char * name,const char * extension="sancov")586 static fd_t CovOpenFile(InternalScopedString *path, bool packed,
587                        const char *name, const char *extension = "sancov") {
588   path->clear();
589   if (!packed) {
590     CHECK(name);
591     path->append("%s/%s.%zd.%s", coverage_dir, name, internal_getpid(),
592                 extension);
593   } else {
594     if (!name)
595       path->append("%s/%zd.%s.packed", coverage_dir, internal_getpid(),
596                   extension);
597     else
598       path->append("%s/%s.%s.packed", coverage_dir, name, extension);
599   }
600   error_t err;
601   fd_t fd = OpenFile(path->data(), WrOnly, &err);
602   if (fd == kInvalidFd)
603     Report("SanitizerCoverage: failed to open %s for writing (reason: %d)\n",
604            path->data(), err);
605   return fd;
606 }
607 
608 // Dump trace PCs and trace events into two separate files.
DumpTrace()609 void CoverageData::DumpTrace() {
610   uptr max_idx = tr_event_pointer - tr_event_array;
611   if (!max_idx) return;
612   auto sym = Symbolizer::GetOrInit();
613   if (!sym)
614     return;
615   InternalScopedString out(32 << 20);
616   for (uptr i = 0, n = size(); i < n; i++) {
617     const char *module_name = "<unknown>";
618     uptr module_address = 0;
619     sym->GetModuleNameAndOffsetForPC(UnbundlePc(pc_array[i]), &module_name,
620                                      &module_address);
621     out.append("%s 0x%zx\n", module_name, module_address);
622   }
623   InternalScopedString path(kMaxPathLength);
624   fd_t fd = CovOpenFile(&path, false, "trace-points");
625   if (fd == kInvalidFd) return;
626   WriteToFile(fd, out.data(), out.length());
627   CloseFile(fd);
628 
629   fd = CovOpenFile(&path, false, "trace-compunits");
630   if (fd == kInvalidFd) return;
631   out.clear();
632   for (uptr i = 0; i < comp_unit_name_vec.size(); i++)
633     out.append("%s\n", comp_unit_name_vec[i].copied_module_name);
634   WriteToFile(fd, out.data(), out.length());
635   CloseFile(fd);
636 
637   fd = CovOpenFile(&path, false, "trace-events");
638   if (fd == kInvalidFd) return;
639   uptr bytes_to_write = max_idx * sizeof(tr_event_array[0]);
640   u8 *event_bytes = reinterpret_cast<u8*>(tr_event_array);
641   // The trace file could be huge, and may not be written with a single syscall.
642   while (bytes_to_write) {
643     uptr actually_written;
644     if (WriteToFile(fd, event_bytes, bytes_to_write, &actually_written) &&
645         actually_written <= bytes_to_write) {
646       bytes_to_write -= actually_written;
647       event_bytes += actually_written;
648     } else {
649       break;
650     }
651   }
652   CloseFile(fd);
653   VReport(1, " CovDump: Trace: %zd PCs written\n", size());
654   VReport(1, " CovDump: Trace: %zd Events written\n", max_idx);
655 }
656 
657 // This function dumps the caller=>callee pairs into a file as a sequence of
658 // lines like "module_name offset".
DumpCallerCalleePairs()659 void CoverageData::DumpCallerCalleePairs() {
660   uptr max_idx = atomic_load(&cc_array_index, memory_order_relaxed);
661   if (!max_idx) return;
662   auto sym = Symbolizer::GetOrInit();
663   if (!sym)
664     return;
665   InternalScopedString out(32 << 20);
666   uptr total = 0;
667   for (uptr i = 0; i < max_idx; i++) {
668     uptr *cc_cache = cc_array[i];
669     CHECK(cc_cache);
670     uptr caller = cc_cache[0];
671     uptr n_callees = cc_cache[1];
672     const char *caller_module_name = "<unknown>";
673     uptr caller_module_address = 0;
674     sym->GetModuleNameAndOffsetForPC(caller, &caller_module_name,
675                                      &caller_module_address);
676     for (uptr j = 2; j < n_callees; j++) {
677       uptr callee = cc_cache[j];
678       if (!callee) break;
679       total++;
680       const char *callee_module_name = "<unknown>";
681       uptr callee_module_address = 0;
682       sym->GetModuleNameAndOffsetForPC(callee, &callee_module_name,
683                                        &callee_module_address);
684       out.append("%s 0x%zx\n%s 0x%zx\n", caller_module_name,
685                  caller_module_address, callee_module_name,
686                  callee_module_address);
687     }
688   }
689   InternalScopedString path(kMaxPathLength);
690   fd_t fd = CovOpenFile(&path, false, "caller-callee");
691   if (fd == kInvalidFd) return;
692   WriteToFile(fd, out.data(), out.length());
693   CloseFile(fd);
694   VReport(1, " CovDump: %zd caller-callee pairs written\n", total);
695 }
696 
697 // Record the current PC into the event buffer.
698 // Every event is a u32 value (index in tr_pc_array_index) so we compute
699 // it once and then cache in the provided 'cache' storage.
700 //
701 // This function will eventually be inlined by the compiler.
TraceBasicBlock(u32 * id)702 void CoverageData::TraceBasicBlock(u32 *id) {
703   // Will trap here if
704   //  1. coverage is not enabled at run-time.
705   //  2. The array tr_event_array is full.
706   *tr_event_pointer = *id - 1;
707   tr_event_pointer++;
708 }
709 
DumpCounters()710 void CoverageData::DumpCounters() {
711   if (!common_flags()->coverage_counters) return;
712   uptr n = coverage_data.GetNumberOf8bitCounters();
713   if (!n) return;
714   InternalScopedBuffer<u8> bitset(n);
715   coverage_data.Update8bitCounterBitsetAndClearCounters(bitset.data());
716   InternalScopedString path(kMaxPathLength);
717 
718   for (uptr m = 0; m < module_name_vec.size(); m++) {
719     auto r = module_name_vec[m];
720     CHECK(r.copied_module_name);
721     CHECK_LE(r.beg, r.end);
722     CHECK_LE(r.end, size());
723     const char *base_name = StripModuleName(r.copied_module_name);
724     fd_t fd =
725         CovOpenFile(&path, /* packed */ false, base_name, "counters-sancov");
726     if (fd == kInvalidFd) return;
727     WriteToFile(fd, bitset.data() + r.beg, r.end - r.beg);
728     CloseFile(fd);
729     VReport(1, " CovDump: %zd counters written for '%s'\n", r.end - r.beg,
730             base_name);
731   }
732 }
733 
DumpAsBitSet()734 void CoverageData::DumpAsBitSet() {
735   if (!common_flags()->coverage_bitset) return;
736   if (!size()) return;
737   InternalScopedBuffer<char> out(size());
738   InternalScopedString path(kMaxPathLength);
739   for (uptr m = 0; m < module_name_vec.size(); m++) {
740     uptr n_set_bits = 0;
741     auto r = module_name_vec[m];
742     CHECK(r.copied_module_name);
743     CHECK_LE(r.beg, r.end);
744     CHECK_LE(r.end, size());
745     for (uptr i = r.beg; i < r.end; i++) {
746       uptr pc = UnbundlePc(pc_array[i]);
747       out[i] = pc ? '1' : '0';
748       if (pc)
749         n_set_bits++;
750     }
751     const char *base_name = StripModuleName(r.copied_module_name);
752     fd_t fd = CovOpenFile(&path, /* packed */false, base_name, "bitset-sancov");
753     if (fd == kInvalidFd) return;
754     WriteToFile(fd, out.data() + r.beg, r.end - r.beg);
755     CloseFile(fd);
756     VReport(1,
757             " CovDump: bitset of %zd bits written for '%s', %zd bits are set\n",
758             r.end - r.beg, base_name, n_set_bits);
759   }
760 }
761 
762 
GetRangeOffsets(const NamedPcRange & r,Symbolizer * sym,InternalMmapVector<uptr> * offsets) const763 void CoverageData::GetRangeOffsets(const NamedPcRange& r, Symbolizer* sym,
764     InternalMmapVector<uptr>* offsets) const {
765   offsets->clear();
766   for (uptr i = 0; i < kNumWordsForMagic; i++)
767     offsets->push_back(0);
768   CHECK(r.copied_module_name);
769   CHECK_LE(r.beg, r.end);
770   CHECK_LE(r.end, size());
771   for (uptr i = r.beg; i < r.end; i++) {
772     uptr pc = UnbundlePc(pc_array[i]);
773     uptr counter = UnbundleCounter(pc_array[i]);
774     if (!pc) continue; // Not visited.
775     uptr offset = 0;
776     sym->GetModuleNameAndOffsetForPC(pc, nullptr, &offset);
777     offsets->push_back(BundlePcAndCounter(offset, counter));
778   }
779 
780   CHECK_GE(offsets->size(), kNumWordsForMagic);
781   SortArray(offsets->data(), offsets->size());
782   for (uptr i = 0; i < offsets->size(); i++)
783     (*offsets)[i] = UnbundlePc((*offsets)[i]);
784 }
785 
GenerateHtmlReport(const InternalMmapVector<char * > & cov_files)786 static void GenerateHtmlReport(const InternalMmapVector<char *> &cov_files) {
787   if (!common_flags()->html_cov_report) {
788     return;
789   }
790   char *sancov_path = FindPathToBinary(common_flags()->sancov_path);
791   if (sancov_path == nullptr) {
792     return;
793   }
794 
795   InternalMmapVector<char *> sancov_argv(cov_files.size() * 2 + 3);
796   sancov_argv.push_back(sancov_path);
797   sancov_argv.push_back(internal_strdup("-html-report"));
798   auto argv_deleter = at_scope_exit([&] {
799     for (uptr i = 0; i < sancov_argv.size(); ++i) {
800       InternalFree(sancov_argv[i]);
801     }
802   });
803 
804   for (const auto &cov_file : cov_files) {
805     sancov_argv.push_back(internal_strdup(cov_file));
806   }
807 
808   {
809     ListOfModules modules;
810     modules.init();
811     for (const LoadedModule &module : modules) {
812       sancov_argv.push_back(internal_strdup(module.full_name()));
813     }
814   }
815 
816   InternalScopedString report_path(kMaxPathLength);
817   fd_t report_fd =
818       CovOpenFile(&report_path, false /* packed */, GetProcessName(), "html");
819   int pid = StartSubprocess(sancov_argv[0], sancov_argv.data(),
820                             kInvalidFd /* stdin */, report_fd /* std_out */);
821   if (pid > 0) {
822     int result = WaitForProcess(pid);
823     if (result == 0)
824       Printf("coverage report generated to %s\n", report_path.data());
825   }
826 }
827 
DumpOffsets()828 void CoverageData::DumpOffsets() {
829   auto sym = Symbolizer::GetOrInit();
830   if (!common_flags()->coverage_pcs) return;
831   CHECK_NE(sym, nullptr);
832   InternalMmapVector<uptr> offsets(0);
833   InternalScopedString path(kMaxPathLength);
834 
835   InternalMmapVector<char *> cov_files(module_name_vec.size());
836   auto cov_files_deleter = at_scope_exit([&] {
837     for (uptr i = 0; i < cov_files.size(); ++i) {
838       InternalFree(cov_files[i]);
839     }
840   });
841 
842   for (uptr m = 0; m < module_name_vec.size(); m++) {
843     auto r = module_name_vec[m];
844     GetRangeOffsets(r, sym, &offsets);
845 
846     uptr num_offsets = offsets.size() - kNumWordsForMagic;
847     u64 *magic_p = reinterpret_cast<u64*>(offsets.data());
848     CHECK_EQ(*magic_p, 0ULL);
849     // FIXME: we may want to write 32-bit offsets even in 64-mode
850     // if all the offsets are small enough.
851     *magic_p = kMagic;
852 
853     const char *module_name = StripModuleName(r.copied_module_name);
854     if (cov_sandboxed) {
855       if (cov_fd != kInvalidFd) {
856         CovWritePacked(internal_getpid(), module_name, offsets.data(),
857                        offsets.size() * sizeof(offsets[0]));
858         VReport(1, " CovDump: %zd PCs written to packed file\n", num_offsets);
859       }
860     } else {
861       // One file per module per process.
862       fd_t fd = CovOpenFile(&path, false /* packed */, module_name);
863       if (fd == kInvalidFd) continue;
864       WriteToFile(fd, offsets.data(), offsets.size() * sizeof(offsets[0]));
865       CloseFile(fd);
866       cov_files.push_back(internal_strdup(path.data()));
867       VReport(1, " CovDump: %s: %zd PCs written\n", path.data(), num_offsets);
868     }
869   }
870   if (cov_fd != kInvalidFd)
871     CloseFile(cov_fd);
872 
873   GenerateHtmlReport(cov_files);
874 }
875 
DumpAll()876 void CoverageData::DumpAll() {
877   if (!coverage_enabled || common_flags()->coverage_direct) return;
878   if (atomic_fetch_add(&dump_once_guard, 1, memory_order_relaxed))
879     return;
880   DumpAsBitSet();
881   DumpCounters();
882   DumpTrace();
883   DumpOffsets();
884   DumpCallerCalleePairs();
885 }
886 
CovPrepareForSandboxing(__sanitizer_sandbox_arguments * args)887 void CovPrepareForSandboxing(__sanitizer_sandbox_arguments *args) {
888   if (!args) return;
889   if (!coverage_enabled) return;
890   cov_sandboxed = args->coverage_sandboxed;
891   if (!cov_sandboxed) return;
892   cov_max_block_size = args->coverage_max_block_size;
893   if (args->coverage_fd >= 0) {
894     cov_fd = (fd_t)args->coverage_fd;
895   } else {
896     InternalScopedString path(kMaxPathLength);
897     // Pre-open the file now. The sandbox won't allow us to do it later.
898     cov_fd = CovOpenFile(&path, true /* packed */, nullptr);
899   }
900 }
901 
MaybeOpenCovFile(const char * name)902 fd_t MaybeOpenCovFile(const char *name) {
903   CHECK(name);
904   if (!coverage_enabled) return kInvalidFd;
905   InternalScopedString path(kMaxPathLength);
906   return CovOpenFile(&path, true /* packed */, name);
907 }
908 
CovBeforeFork()909 void CovBeforeFork() {
910   coverage_data.BeforeFork();
911 }
912 
CovAfterFork(int child_pid)913 void CovAfterFork(int child_pid) {
914   coverage_data.AfterFork(child_pid);
915 }
916 
MaybeDumpCoverage()917 static void MaybeDumpCoverage() {
918   if (common_flags()->coverage)
919     __sanitizer_cov_dump();
920 }
921 
InitializeCoverage(bool enabled,const char * dir)922 void InitializeCoverage(bool enabled, const char *dir) {
923   if (coverage_enabled)
924     return;  // May happen if two sanitizer enable coverage in the same process.
925   coverage_enabled = enabled;
926   coverage_dir = dir;
927   coverage_data.Init();
928   if (enabled) coverage_data.Enable();
929   if (!common_flags()->coverage_direct) Atexit(__sanitizer_cov_dump);
930   AddDieCallback(MaybeDumpCoverage);
931 }
932 
ReInitializeCoverage(bool enabled,const char * dir)933 void ReInitializeCoverage(bool enabled, const char *dir) {
934   coverage_enabled = enabled;
935   coverage_dir = dir;
936   coverage_data.ReInit();
937 }
938 
CoverageUpdateMapping()939 void CoverageUpdateMapping() {
940   if (coverage_enabled)
941     CovUpdateMapping(coverage_dir);
942 }
943 
944 } // namespace __sanitizer
945 
946 extern "C" {
__sanitizer_cov(u32 * guard)947 SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov(u32 *guard) {
948   coverage_data.Add(StackTrace::GetPreviousInstructionPc(GET_CALLER_PC()),
949                     guard);
950 }
__sanitizer_cov_with_check(u32 * guard)951 SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_with_check(u32 *guard) {
952   atomic_uint32_t *atomic_guard = reinterpret_cast<atomic_uint32_t*>(guard);
953   if (static_cast<s32>(
954           __sanitizer::atomic_load(atomic_guard, memory_order_relaxed)) < 0)
955     __sanitizer_cov(guard);
956 }
957 SANITIZER_INTERFACE_ATTRIBUTE void
__sanitizer_cov_indir_call16(uptr callee,uptr callee_cache16[])958 __sanitizer_cov_indir_call16(uptr callee, uptr callee_cache16[]) {
959   coverage_data.IndirCall(StackTrace::GetPreviousInstructionPc(GET_CALLER_PC()),
960                           callee, callee_cache16, 16);
961 }
__sanitizer_cov_init()962 SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_init() {
963   coverage_enabled = true;
964   coverage_dir = common_flags()->coverage_dir;
965   coverage_data.Init();
966 }
__sanitizer_cov_dump()967 SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_dump() {
968   coverage_data.DumpAll();
969 }
970 SANITIZER_INTERFACE_ATTRIBUTE void
__sanitizer_cov_module_init(s32 * guards,uptr npcs,u8 * counters,const char * comp_unit_name)971 __sanitizer_cov_module_init(s32 *guards, uptr npcs, u8 *counters,
972                             const char *comp_unit_name) {
973   coverage_data.InitializeGuards(guards, npcs, comp_unit_name, GET_CALLER_PC());
974   coverage_data.InitializeCounters(counters, npcs);
975   if (!common_flags()->coverage_direct) return;
976   if (SANITIZER_ANDROID && coverage_enabled) {
977     // dlopen/dlclose interceptors do not work on Android, so we rely on
978     // Extend() calls to update .sancov.map.
979     CovUpdateMapping(coverage_dir, GET_CALLER_PC());
980   }
981   coverage_data.Extend(npcs);
982 }
983 SANITIZER_INTERFACE_ATTRIBUTE
__sanitizer_maybe_open_cov_file(const char * name)984 sptr __sanitizer_maybe_open_cov_file(const char *name) {
985   return (sptr)MaybeOpenCovFile(name);
986 }
987 SANITIZER_INTERFACE_ATTRIBUTE
__sanitizer_get_total_unique_coverage()988 uptr __sanitizer_get_total_unique_coverage() {
989   return atomic_load(&coverage_counter, memory_order_relaxed);
990 }
991 
992 SANITIZER_INTERFACE_ATTRIBUTE
__sanitizer_get_total_unique_caller_callee_pairs()993 uptr __sanitizer_get_total_unique_caller_callee_pairs() {
994   return atomic_load(&caller_callee_counter, memory_order_relaxed);
995 }
996 
997 SANITIZER_INTERFACE_ATTRIBUTE
__sanitizer_cov_trace_func_enter(u32 * id)998 void __sanitizer_cov_trace_func_enter(u32 *id) {
999   __sanitizer_cov_with_check(id);
1000   coverage_data.TraceBasicBlock(id);
1001 }
1002 SANITIZER_INTERFACE_ATTRIBUTE
__sanitizer_cov_trace_basic_block(u32 * id)1003 void __sanitizer_cov_trace_basic_block(u32 *id) {
1004   __sanitizer_cov_with_check(id);
1005   coverage_data.TraceBasicBlock(id);
1006 }
1007 SANITIZER_INTERFACE_ATTRIBUTE
__sanitizer_reset_coverage()1008 void __sanitizer_reset_coverage() {
1009   ResetGlobalCounters();
1010   coverage_data.ReinitializeGuards();
1011   internal_bzero_aligned16(
1012       coverage_data.data(),
1013       RoundUpTo(coverage_data.size() * sizeof(coverage_data.data()[0]), 16));
1014 }
1015 SANITIZER_INTERFACE_ATTRIBUTE
__sanitizer_get_coverage_guards(uptr ** data)1016 uptr __sanitizer_get_coverage_guards(uptr **data) {
1017   *data = coverage_data.data();
1018   return coverage_data.size();
1019 }
1020 
1021 SANITIZER_INTERFACE_ATTRIBUTE
__sanitizer_get_coverage_pc_buffer(uptr ** data)1022 uptr __sanitizer_get_coverage_pc_buffer(uptr **data) {
1023   *data = coverage_data.buffer();
1024   return __sanitizer_get_total_unique_coverage();
1025 }
1026 
1027 SANITIZER_INTERFACE_ATTRIBUTE
__sanitizer_get_number_of_counters()1028 uptr __sanitizer_get_number_of_counters() {
1029   return coverage_data.GetNumberOf8bitCounters();
1030 }
1031 
1032 SANITIZER_INTERFACE_ATTRIBUTE
__sanitizer_update_counter_bitset_and_clear_counters(u8 * bitset)1033 uptr __sanitizer_update_counter_bitset_and_clear_counters(u8 *bitset) {
1034   return coverage_data.Update8bitCounterBitsetAndClearCounters(bitset);
1035 }
1036 // Default empty implementations (weak). Users should redefine them.
1037 SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
__sanitizer_cov_trace_cmp()1038 void __sanitizer_cov_trace_cmp() {}
1039 SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
__sanitizer_cov_trace_switch()1040 void __sanitizer_cov_trace_switch() {}
1041 } // extern "C"
1042