1 //===-- MemorySanitizer.cpp - detector of uninitialized reads -------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 /// \file
10 /// This file is a part of MemorySanitizer, a detector of uninitialized
11 /// reads.
12 ///
13 /// The algorithm of the tool is similar to Memcheck
14 /// (http://goo.gl/QKbem). We associate a few shadow bits with every
15 /// byte of the application memory, poison the shadow of the malloc-ed
16 /// or alloca-ed memory, load the shadow bits on every memory read,
17 /// propagate the shadow bits through some of the arithmetic
18 /// instruction (including MOV), store the shadow bits on every memory
19 /// write, report a bug on some other instructions (e.g. JMP) if the
20 /// associated shadow is poisoned.
21 ///
22 /// But there are differences too. The first and the major one:
23 /// compiler instrumentation instead of binary instrumentation. This
24 /// gives us much better register allocation, possible compiler
25 /// optimizations and a fast start-up. But this brings the major issue
26 /// as well: msan needs to see all program events, including system
27 /// calls and reads/writes in system libraries, so we either need to
28 /// compile *everything* with msan or use a binary translation
29 /// component (e.g. DynamoRIO) to instrument pre-built libraries.
30 /// Another difference from Memcheck is that we use 8 shadow bits per
31 /// byte of application memory and use a direct shadow mapping. This
32 /// greatly simplifies the instrumentation code and avoids races on
33 /// shadow updates (Memcheck is single-threaded so races are not a
34 /// concern there. Memcheck uses 2 shadow bits per byte with a slow
35 /// path storage that uses 8 bits per byte).
36 ///
37 /// The default value of shadow is 0, which means "clean" (not poisoned).
38 ///
39 /// Every module initializer should call __msan_init to ensure that the
40 /// shadow memory is ready. On error, __msan_warning is called. Since
41 /// parameters and return values may be passed via registers, we have a
42 /// specialized thread-local shadow for return values
43 /// (__msan_retval_tls) and parameters (__msan_param_tls).
44 ///
45 /// Origin tracking.
46 ///
47 /// MemorySanitizer can track origins (allocation points) of all uninitialized
48 /// values. This behavior is controlled with a flag (msan-track-origins) and is
49 /// disabled by default.
50 ///
51 /// Origins are 4-byte values created and interpreted by the runtime library.
52 /// They are stored in a second shadow mapping, one 4-byte value for 4 bytes
53 /// of application memory. Propagation of origins is basically a bunch of
54 /// "select" instructions that pick the origin of a dirty argument, if an
55 /// instruction has one.
56 ///
57 /// Every 4 aligned, consecutive bytes of application memory have one origin
58 /// value associated with them. If these bytes contain uninitialized data
59 /// coming from 2 different allocations, the last store wins. Because of this,
60 /// MemorySanitizer reports can show unrelated origins, but this is unlikely in
61 /// practice.
62 ///
63 /// Origins are meaningless for fully initialized values, so MemorySanitizer
64 /// avoids storing origin to memory when a fully initialized value is stored.
65 /// This way it avoids needless overwritting origin of the 4-byte region on
66 /// a short (i.e. 1 byte) clean store, and it is also good for performance.
67 ///
68 /// Atomic handling.
69 ///
70 /// Ideally, every atomic store of application value should update the
71 /// corresponding shadow location in an atomic way. Unfortunately, atomic store
72 /// of two disjoint locations can not be done without severe slowdown.
73 ///
74 /// Therefore, we implement an approximation that may err on the safe side.
75 /// In this implementation, every atomically accessed location in the program
76 /// may only change from (partially) uninitialized to fully initialized, but
77 /// not the other way around. We load the shadow _after_ the application load,
78 /// and we store the shadow _before_ the app store. Also, we always store clean
79 /// shadow (if the application store is atomic). This way, if the store-load
80 /// pair constitutes a happens-before arc, shadow store and load are correctly
81 /// ordered such that the load will get either the value that was stored, or
82 /// some later value (which is always clean).
83 ///
84 /// This does not work very well with Compare-And-Swap (CAS) and
85 /// Read-Modify-Write (RMW) operations. To follow the above logic, CAS and RMW
86 /// must store the new shadow before the app operation, and load the shadow
87 /// after the app operation. Computers don't work this way. Current
88 /// implementation ignores the load aspect of CAS/RMW, always returning a clean
89 /// value. It implements the store part as a simple atomic store by storing a
90 /// clean shadow.
91
92 //===----------------------------------------------------------------------===//
93
94 #include "llvm/ADT/DepthFirstIterator.h"
95 #include "llvm/ADT/SmallString.h"
96 #include "llvm/ADT/SmallVector.h"
97 #include "llvm/ADT/StringExtras.h"
98 #include "llvm/ADT/Triple.h"
99 #include "llvm/IR/DataLayout.h"
100 #include "llvm/IR/Function.h"
101 #include "llvm/IR/IRBuilder.h"
102 #include "llvm/IR/InlineAsm.h"
103 #include "llvm/IR/InstVisitor.h"
104 #include "llvm/IR/IntrinsicInst.h"
105 #include "llvm/IR/LLVMContext.h"
106 #include "llvm/IR/MDBuilder.h"
107 #include "llvm/IR/Module.h"
108 #include "llvm/IR/Type.h"
109 #include "llvm/IR/ValueMap.h"
110 #include "llvm/Support/CommandLine.h"
111 #include "llvm/Support/Debug.h"
112 #include "llvm/Support/raw_ostream.h"
113 #include "llvm/Transforms/Instrumentation.h"
114 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
115 #include "llvm/Transforms/Utils/Local.h"
116 #include "llvm/Transforms/Utils/ModuleUtils.h"
117
118 using namespace llvm;
119
120 #define DEBUG_TYPE "msan"
121
122 static const unsigned kOriginSize = 4;
123 static const unsigned kMinOriginAlignment = 4;
124 static const unsigned kShadowTLSAlignment = 8;
125
126 // These constants must be kept in sync with the ones in msan.h.
127 static const unsigned kParamTLSSize = 800;
128 static const unsigned kRetvalTLSSize = 800;
129
130 // Accesses sizes are powers of two: 1, 2, 4, 8.
131 static const size_t kNumberOfAccessSizes = 4;
132
133 /// \brief Track origins of uninitialized values.
134 ///
135 /// Adds a section to MemorySanitizer report that points to the allocation
136 /// (stack or heap) the uninitialized bits came from originally.
137 static cl::opt<int> ClTrackOrigins("msan-track-origins",
138 cl::desc("Track origins (allocation sites) of poisoned memory"),
139 cl::Hidden, cl::init(0));
140 static cl::opt<bool> ClKeepGoing("msan-keep-going",
141 cl::desc("keep going after reporting a UMR"),
142 cl::Hidden, cl::init(false));
143 static cl::opt<bool> ClPoisonStack("msan-poison-stack",
144 cl::desc("poison uninitialized stack variables"),
145 cl::Hidden, cl::init(true));
146 static cl::opt<bool> ClPoisonStackWithCall("msan-poison-stack-with-call",
147 cl::desc("poison uninitialized stack variables with a call"),
148 cl::Hidden, cl::init(false));
149 static cl::opt<int> ClPoisonStackPattern("msan-poison-stack-pattern",
150 cl::desc("poison uninitialized stack variables with the given pattern"),
151 cl::Hidden, cl::init(0xff));
152 static cl::opt<bool> ClPoisonUndef("msan-poison-undef",
153 cl::desc("poison undef temps"),
154 cl::Hidden, cl::init(true));
155
156 static cl::opt<bool> ClHandleICmp("msan-handle-icmp",
157 cl::desc("propagate shadow through ICmpEQ and ICmpNE"),
158 cl::Hidden, cl::init(true));
159
160 static cl::opt<bool> ClHandleICmpExact("msan-handle-icmp-exact",
161 cl::desc("exact handling of relational integer ICmp"),
162 cl::Hidden, cl::init(false));
163
164 // This flag controls whether we check the shadow of the address
165 // operand of load or store. Such bugs are very rare, since load from
166 // a garbage address typically results in SEGV, but still happen
167 // (e.g. only lower bits of address are garbage, or the access happens
168 // early at program startup where malloc-ed memory is more likely to
169 // be zeroed. As of 2012-08-28 this flag adds 20% slowdown.
170 static cl::opt<bool> ClCheckAccessAddress("msan-check-access-address",
171 cl::desc("report accesses through a pointer which has poisoned shadow"),
172 cl::Hidden, cl::init(true));
173
174 static cl::opt<bool> ClDumpStrictInstructions("msan-dump-strict-instructions",
175 cl::desc("print out instructions with default strict semantics"),
176 cl::Hidden, cl::init(false));
177
178 static cl::opt<int> ClInstrumentationWithCallThreshold(
179 "msan-instrumentation-with-call-threshold",
180 cl::desc(
181 "If the function being instrumented requires more than "
182 "this number of checks and origin stores, use callbacks instead of "
183 "inline checks (-1 means never use callbacks)."),
184 cl::Hidden, cl::init(3500));
185
186 // This is an experiment to enable handling of cases where shadow is a non-zero
187 // compile-time constant. For some unexplainable reason they were silently
188 // ignored in the instrumentation.
189 static cl::opt<bool> ClCheckConstantShadow("msan-check-constant-shadow",
190 cl::desc("Insert checks for constant shadow values"),
191 cl::Hidden, cl::init(false));
192
193 // This is off by default because of a bug in gold:
194 // https://sourceware.org/bugzilla/show_bug.cgi?id=19002
195 static cl::opt<bool> ClWithComdat("msan-with-comdat",
196 cl::desc("Place MSan constructors in comdat sections"),
197 cl::Hidden, cl::init(false));
198
199 static const char *const kMsanModuleCtorName = "msan.module_ctor";
200 static const char *const kMsanInitName = "__msan_init";
201
202 namespace {
203
204 // Memory map parameters used in application-to-shadow address calculation.
205 // Offset = (Addr & ~AndMask) ^ XorMask
206 // Shadow = ShadowBase + Offset
207 // Origin = OriginBase + Offset
208 struct MemoryMapParams {
209 uint64_t AndMask;
210 uint64_t XorMask;
211 uint64_t ShadowBase;
212 uint64_t OriginBase;
213 };
214
215 struct PlatformMemoryMapParams {
216 const MemoryMapParams *bits32;
217 const MemoryMapParams *bits64;
218 };
219
220 // i386 Linux
221 static const MemoryMapParams Linux_I386_MemoryMapParams = {
222 0x000080000000, // AndMask
223 0, // XorMask (not used)
224 0, // ShadowBase (not used)
225 0x000040000000, // OriginBase
226 };
227
228 // x86_64 Linux
229 static const MemoryMapParams Linux_X86_64_MemoryMapParams = {
230 #ifdef MSAN_LINUX_X86_64_OLD_MAPPING
231 0x400000000000, // AndMask
232 0, // XorMask (not used)
233 0, // ShadowBase (not used)
234 0x200000000000, // OriginBase
235 #else
236 0, // AndMask (not used)
237 0x500000000000, // XorMask
238 0, // ShadowBase (not used)
239 0x100000000000, // OriginBase
240 #endif
241 };
242
243 // mips64 Linux
244 static const MemoryMapParams Linux_MIPS64_MemoryMapParams = {
245 0x004000000000, // AndMask
246 0, // XorMask (not used)
247 0, // ShadowBase (not used)
248 0x002000000000, // OriginBase
249 };
250
251 // ppc64 Linux
252 static const MemoryMapParams Linux_PowerPC64_MemoryMapParams = {
253 0x200000000000, // AndMask
254 0x100000000000, // XorMask
255 0x080000000000, // ShadowBase
256 0x1C0000000000, // OriginBase
257 };
258
259 // aarch64 Linux
260 static const MemoryMapParams Linux_AArch64_MemoryMapParams = {
261 0, // AndMask (not used)
262 0x06000000000, // XorMask
263 0, // ShadowBase (not used)
264 0x01000000000, // OriginBase
265 };
266
267 // i386 FreeBSD
268 static const MemoryMapParams FreeBSD_I386_MemoryMapParams = {
269 0x000180000000, // AndMask
270 0x000040000000, // XorMask
271 0x000020000000, // ShadowBase
272 0x000700000000, // OriginBase
273 };
274
275 // x86_64 FreeBSD
276 static const MemoryMapParams FreeBSD_X86_64_MemoryMapParams = {
277 0xc00000000000, // AndMask
278 0x200000000000, // XorMask
279 0x100000000000, // ShadowBase
280 0x380000000000, // OriginBase
281 };
282
283 static const PlatformMemoryMapParams Linux_X86_MemoryMapParams = {
284 &Linux_I386_MemoryMapParams,
285 &Linux_X86_64_MemoryMapParams,
286 };
287
288 static const PlatformMemoryMapParams Linux_MIPS_MemoryMapParams = {
289 nullptr,
290 &Linux_MIPS64_MemoryMapParams,
291 };
292
293 static const PlatformMemoryMapParams Linux_PowerPC_MemoryMapParams = {
294 nullptr,
295 &Linux_PowerPC64_MemoryMapParams,
296 };
297
298 static const PlatformMemoryMapParams Linux_ARM_MemoryMapParams = {
299 nullptr,
300 &Linux_AArch64_MemoryMapParams,
301 };
302
303 static const PlatformMemoryMapParams FreeBSD_X86_MemoryMapParams = {
304 &FreeBSD_I386_MemoryMapParams,
305 &FreeBSD_X86_64_MemoryMapParams,
306 };
307
308 /// \brief An instrumentation pass implementing detection of uninitialized
309 /// reads.
310 ///
311 /// MemorySanitizer: instrument the code in module to find
312 /// uninitialized reads.
313 class MemorySanitizer : public FunctionPass {
314 public:
MemorySanitizer(int TrackOrigins=0)315 MemorySanitizer(int TrackOrigins = 0)
316 : FunctionPass(ID),
317 TrackOrigins(std::max(TrackOrigins, (int)ClTrackOrigins)),
318 WarningFn(nullptr) {}
getPassName() const319 const char *getPassName() const override { return "MemorySanitizer"; }
getAnalysisUsage(AnalysisUsage & AU) const320 void getAnalysisUsage(AnalysisUsage &AU) const override {
321 AU.addRequired<TargetLibraryInfoWrapperPass>();
322 }
323 bool runOnFunction(Function &F) override;
324 bool doInitialization(Module &M) override;
325 static char ID; // Pass identification, replacement for typeid.
326
327 private:
328 void initializeCallbacks(Module &M);
329
330 /// \brief Track origins (allocation points) of uninitialized values.
331 int TrackOrigins;
332
333 LLVMContext *C;
334 Type *IntptrTy;
335 Type *OriginTy;
336 /// \brief Thread-local shadow storage for function parameters.
337 GlobalVariable *ParamTLS;
338 /// \brief Thread-local origin storage for function parameters.
339 GlobalVariable *ParamOriginTLS;
340 /// \brief Thread-local shadow storage for function return value.
341 GlobalVariable *RetvalTLS;
342 /// \brief Thread-local origin storage for function return value.
343 GlobalVariable *RetvalOriginTLS;
344 /// \brief Thread-local shadow storage for in-register va_arg function
345 /// parameters (x86_64-specific).
346 GlobalVariable *VAArgTLS;
347 /// \brief Thread-local shadow storage for va_arg overflow area
348 /// (x86_64-specific).
349 GlobalVariable *VAArgOverflowSizeTLS;
350 /// \brief Thread-local space used to pass origin value to the UMR reporting
351 /// function.
352 GlobalVariable *OriginTLS;
353
354 /// \brief The run-time callback to print a warning.
355 Value *WarningFn;
356 // These arrays are indexed by log2(AccessSize).
357 Value *MaybeWarningFn[kNumberOfAccessSizes];
358 Value *MaybeStoreOriginFn[kNumberOfAccessSizes];
359
360 /// \brief Run-time helper that generates a new origin value for a stack
361 /// allocation.
362 Value *MsanSetAllocaOrigin4Fn;
363 /// \brief Run-time helper that poisons stack on function entry.
364 Value *MsanPoisonStackFn;
365 /// \brief Run-time helper that records a store (or any event) of an
366 /// uninitialized value and returns an updated origin id encoding this info.
367 Value *MsanChainOriginFn;
368 /// \brief MSan runtime replacements for memmove, memcpy and memset.
369 Value *MemmoveFn, *MemcpyFn, *MemsetFn;
370
371 /// \brief Memory map parameters used in application-to-shadow calculation.
372 const MemoryMapParams *MapParams;
373
374 MDNode *ColdCallWeights;
375 /// \brief Branch weights for origin store.
376 MDNode *OriginStoreWeights;
377 /// \brief An empty volatile inline asm that prevents callback merge.
378 InlineAsm *EmptyAsm;
379 Function *MsanCtorFunction;
380
381 friend struct MemorySanitizerVisitor;
382 friend struct VarArgAMD64Helper;
383 friend struct VarArgMIPS64Helper;
384 friend struct VarArgAArch64Helper;
385 friend struct VarArgPowerPC64Helper;
386 };
387 } // anonymous namespace
388
389 char MemorySanitizer::ID = 0;
390 INITIALIZE_PASS_BEGIN(
391 MemorySanitizer, "msan",
392 "MemorySanitizer: detects uninitialized reads.", false, false)
INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)393 INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
394 INITIALIZE_PASS_END(
395 MemorySanitizer, "msan",
396 "MemorySanitizer: detects uninitialized reads.", false, false)
397
398 FunctionPass *llvm::createMemorySanitizerPass(int TrackOrigins) {
399 return new MemorySanitizer(TrackOrigins);
400 }
401
402 /// \brief Create a non-const global initialized with the given string.
403 ///
404 /// Creates a writable global for Str so that we can pass it to the
405 /// run-time lib. Runtime uses first 4 bytes of the string to store the
406 /// frame ID, so the string needs to be mutable.
createPrivateNonConstGlobalForString(Module & M,StringRef Str)407 static GlobalVariable *createPrivateNonConstGlobalForString(Module &M,
408 StringRef Str) {
409 Constant *StrConst = ConstantDataArray::getString(M.getContext(), Str);
410 return new GlobalVariable(M, StrConst->getType(), /*isConstant=*/false,
411 GlobalValue::PrivateLinkage, StrConst, "");
412 }
413
414 /// \brief Insert extern declaration of runtime-provided functions and globals.
initializeCallbacks(Module & M)415 void MemorySanitizer::initializeCallbacks(Module &M) {
416 // Only do this once.
417 if (WarningFn)
418 return;
419
420 IRBuilder<> IRB(*C);
421 // Create the callback.
422 // FIXME: this function should have "Cold" calling conv,
423 // which is not yet implemented.
424 StringRef WarningFnName = ClKeepGoing ? "__msan_warning"
425 : "__msan_warning_noreturn";
426 WarningFn = M.getOrInsertFunction(WarningFnName, IRB.getVoidTy(), nullptr);
427
428 for (size_t AccessSizeIndex = 0; AccessSizeIndex < kNumberOfAccessSizes;
429 AccessSizeIndex++) {
430 unsigned AccessSize = 1 << AccessSizeIndex;
431 std::string FunctionName = "__msan_maybe_warning_" + itostr(AccessSize);
432 MaybeWarningFn[AccessSizeIndex] = M.getOrInsertFunction(
433 FunctionName, IRB.getVoidTy(), IRB.getIntNTy(AccessSize * 8),
434 IRB.getInt32Ty(), nullptr);
435
436 FunctionName = "__msan_maybe_store_origin_" + itostr(AccessSize);
437 MaybeStoreOriginFn[AccessSizeIndex] = M.getOrInsertFunction(
438 FunctionName, IRB.getVoidTy(), IRB.getIntNTy(AccessSize * 8),
439 IRB.getInt8PtrTy(), IRB.getInt32Ty(), nullptr);
440 }
441
442 MsanSetAllocaOrigin4Fn = M.getOrInsertFunction(
443 "__msan_set_alloca_origin4", IRB.getVoidTy(), IRB.getInt8PtrTy(), IntptrTy,
444 IRB.getInt8PtrTy(), IntptrTy, nullptr);
445 MsanPoisonStackFn =
446 M.getOrInsertFunction("__msan_poison_stack", IRB.getVoidTy(),
447 IRB.getInt8PtrTy(), IntptrTy, nullptr);
448 MsanChainOriginFn = M.getOrInsertFunction(
449 "__msan_chain_origin", IRB.getInt32Ty(), IRB.getInt32Ty(), nullptr);
450 MemmoveFn = M.getOrInsertFunction(
451 "__msan_memmove", IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
452 IRB.getInt8PtrTy(), IntptrTy, nullptr);
453 MemcpyFn = M.getOrInsertFunction(
454 "__msan_memcpy", IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
455 IntptrTy, nullptr);
456 MemsetFn = M.getOrInsertFunction(
457 "__msan_memset", IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IRB.getInt32Ty(),
458 IntptrTy, nullptr);
459
460 // Create globals.
461 RetvalTLS = new GlobalVariable(
462 M, ArrayType::get(IRB.getInt64Ty(), kRetvalTLSSize / 8), false,
463 GlobalVariable::ExternalLinkage, nullptr, "__msan_retval_tls", nullptr,
464 GlobalVariable::InitialExecTLSModel);
465 RetvalOriginTLS = new GlobalVariable(
466 M, OriginTy, false, GlobalVariable::ExternalLinkage, nullptr,
467 "__msan_retval_origin_tls", nullptr, GlobalVariable::InitialExecTLSModel);
468
469 ParamTLS = new GlobalVariable(
470 M, ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8), false,
471 GlobalVariable::ExternalLinkage, nullptr, "__msan_param_tls", nullptr,
472 GlobalVariable::InitialExecTLSModel);
473 ParamOriginTLS = new GlobalVariable(
474 M, ArrayType::get(OriginTy, kParamTLSSize / 4), false,
475 GlobalVariable::ExternalLinkage, nullptr, "__msan_param_origin_tls",
476 nullptr, GlobalVariable::InitialExecTLSModel);
477
478 VAArgTLS = new GlobalVariable(
479 M, ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8), false,
480 GlobalVariable::ExternalLinkage, nullptr, "__msan_va_arg_tls", nullptr,
481 GlobalVariable::InitialExecTLSModel);
482 VAArgOverflowSizeTLS = new GlobalVariable(
483 M, IRB.getInt64Ty(), false, GlobalVariable::ExternalLinkage, nullptr,
484 "__msan_va_arg_overflow_size_tls", nullptr,
485 GlobalVariable::InitialExecTLSModel);
486 OriginTLS = new GlobalVariable(
487 M, IRB.getInt32Ty(), false, GlobalVariable::ExternalLinkage, nullptr,
488 "__msan_origin_tls", nullptr, GlobalVariable::InitialExecTLSModel);
489
490 // We insert an empty inline asm after __msan_report* to avoid callback merge.
491 EmptyAsm = InlineAsm::get(FunctionType::get(IRB.getVoidTy(), false),
492 StringRef(""), StringRef(""),
493 /*hasSideEffects=*/true);
494 }
495
496 /// \brief Module-level initialization.
497 ///
498 /// inserts a call to __msan_init to the module's constructor list.
doInitialization(Module & M)499 bool MemorySanitizer::doInitialization(Module &M) {
500 auto &DL = M.getDataLayout();
501
502 Triple TargetTriple(M.getTargetTriple());
503 switch (TargetTriple.getOS()) {
504 case Triple::FreeBSD:
505 switch (TargetTriple.getArch()) {
506 case Triple::x86_64:
507 MapParams = FreeBSD_X86_MemoryMapParams.bits64;
508 break;
509 case Triple::x86:
510 MapParams = FreeBSD_X86_MemoryMapParams.bits32;
511 break;
512 default:
513 report_fatal_error("unsupported architecture");
514 }
515 break;
516 case Triple::Linux:
517 switch (TargetTriple.getArch()) {
518 case Triple::x86_64:
519 MapParams = Linux_X86_MemoryMapParams.bits64;
520 break;
521 case Triple::x86:
522 MapParams = Linux_X86_MemoryMapParams.bits32;
523 break;
524 case Triple::mips64:
525 case Triple::mips64el:
526 MapParams = Linux_MIPS_MemoryMapParams.bits64;
527 break;
528 case Triple::ppc64:
529 case Triple::ppc64le:
530 MapParams = Linux_PowerPC_MemoryMapParams.bits64;
531 break;
532 case Triple::aarch64:
533 case Triple::aarch64_be:
534 MapParams = Linux_ARM_MemoryMapParams.bits64;
535 break;
536 default:
537 report_fatal_error("unsupported architecture");
538 }
539 break;
540 default:
541 report_fatal_error("unsupported operating system");
542 }
543
544 C = &(M.getContext());
545 IRBuilder<> IRB(*C);
546 IntptrTy = IRB.getIntPtrTy(DL);
547 OriginTy = IRB.getInt32Ty();
548
549 ColdCallWeights = MDBuilder(*C).createBranchWeights(1, 1000);
550 OriginStoreWeights = MDBuilder(*C).createBranchWeights(1, 1000);
551
552 std::tie(MsanCtorFunction, std::ignore) =
553 createSanitizerCtorAndInitFunctions(M, kMsanModuleCtorName, kMsanInitName,
554 /*InitArgTypes=*/{},
555 /*InitArgs=*/{});
556 if (ClWithComdat) {
557 Comdat *MsanCtorComdat = M.getOrInsertComdat(kMsanModuleCtorName);
558 MsanCtorFunction->setComdat(MsanCtorComdat);
559 appendToGlobalCtors(M, MsanCtorFunction, 0, MsanCtorFunction);
560 } else {
561 appendToGlobalCtors(M, MsanCtorFunction, 0);
562 }
563
564
565 if (TrackOrigins)
566 new GlobalVariable(M, IRB.getInt32Ty(), true, GlobalValue::WeakODRLinkage,
567 IRB.getInt32(TrackOrigins), "__msan_track_origins");
568
569 if (ClKeepGoing)
570 new GlobalVariable(M, IRB.getInt32Ty(), true, GlobalValue::WeakODRLinkage,
571 IRB.getInt32(ClKeepGoing), "__msan_keep_going");
572
573 return true;
574 }
575
576 namespace {
577
578 /// \brief A helper class that handles instrumentation of VarArg
579 /// functions on a particular platform.
580 ///
581 /// Implementations are expected to insert the instrumentation
582 /// necessary to propagate argument shadow through VarArg function
583 /// calls. Visit* methods are called during an InstVisitor pass over
584 /// the function, and should avoid creating new basic blocks. A new
585 /// instance of this class is created for each instrumented function.
586 struct VarArgHelper {
587 /// \brief Visit a CallSite.
588 virtual void visitCallSite(CallSite &CS, IRBuilder<> &IRB) = 0;
589
590 /// \brief Visit a va_start call.
591 virtual void visitVAStartInst(VAStartInst &I) = 0;
592
593 /// \brief Visit a va_copy call.
594 virtual void visitVACopyInst(VACopyInst &I) = 0;
595
596 /// \brief Finalize function instrumentation.
597 ///
598 /// This method is called after visiting all interesting (see above)
599 /// instructions in a function.
600 virtual void finalizeInstrumentation() = 0;
601
~VarArgHelper__anon69a70b300211::VarArgHelper602 virtual ~VarArgHelper() {}
603 };
604
605 struct MemorySanitizerVisitor;
606
607 VarArgHelper*
608 CreateVarArgHelper(Function &Func, MemorySanitizer &Msan,
609 MemorySanitizerVisitor &Visitor);
610
TypeSizeToSizeIndex(unsigned TypeSize)611 unsigned TypeSizeToSizeIndex(unsigned TypeSize) {
612 if (TypeSize <= 8) return 0;
613 return Log2_32_Ceil((TypeSize + 7) / 8);
614 }
615
616 /// This class does all the work for a given function. Store and Load
617 /// instructions store and load corresponding shadow and origin
618 /// values. Most instructions propagate shadow from arguments to their
619 /// return values. Certain instructions (most importantly, BranchInst)
620 /// test their argument shadow and print reports (with a runtime call) if it's
621 /// non-zero.
622 struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
623 Function &F;
624 MemorySanitizer &MS;
625 SmallVector<PHINode *, 16> ShadowPHINodes, OriginPHINodes;
626 ValueMap<Value*, Value*> ShadowMap, OriginMap;
627 std::unique_ptr<VarArgHelper> VAHelper;
628 const TargetLibraryInfo *TLI;
629
630 // The following flags disable parts of MSan instrumentation based on
631 // blacklist contents and command-line options.
632 bool InsertChecks;
633 bool PropagateShadow;
634 bool PoisonStack;
635 bool PoisonUndef;
636 bool CheckReturnValue;
637
638 struct ShadowOriginAndInsertPoint {
639 Value *Shadow;
640 Value *Origin;
641 Instruction *OrigIns;
ShadowOriginAndInsertPoint__anon69a70b300211::MemorySanitizerVisitor::ShadowOriginAndInsertPoint642 ShadowOriginAndInsertPoint(Value *S, Value *O, Instruction *I)
643 : Shadow(S), Origin(O), OrigIns(I) { }
644 };
645 SmallVector<ShadowOriginAndInsertPoint, 16> InstrumentationList;
646 SmallVector<StoreInst *, 16> StoreList;
647
MemorySanitizerVisitor__anon69a70b300211::MemorySanitizerVisitor648 MemorySanitizerVisitor(Function &F, MemorySanitizer &MS)
649 : F(F), MS(MS), VAHelper(CreateVarArgHelper(F, MS, *this)) {
650 bool SanitizeFunction = F.hasFnAttribute(Attribute::SanitizeMemory);
651 InsertChecks = SanitizeFunction;
652 PropagateShadow = SanitizeFunction;
653 PoisonStack = SanitizeFunction && ClPoisonStack;
654 PoisonUndef = SanitizeFunction && ClPoisonUndef;
655 // FIXME: Consider using SpecialCaseList to specify a list of functions that
656 // must always return fully initialized values. For now, we hardcode "main".
657 CheckReturnValue = SanitizeFunction && (F.getName() == "main");
658 TLI = &MS.getAnalysis<TargetLibraryInfoWrapperPass>().getTLI();
659
660 DEBUG(if (!InsertChecks)
661 dbgs() << "MemorySanitizer is not inserting checks into '"
662 << F.getName() << "'\n");
663 }
664
updateOrigin__anon69a70b300211::MemorySanitizerVisitor665 Value *updateOrigin(Value *V, IRBuilder<> &IRB) {
666 if (MS.TrackOrigins <= 1) return V;
667 return IRB.CreateCall(MS.MsanChainOriginFn, V);
668 }
669
originToIntptr__anon69a70b300211::MemorySanitizerVisitor670 Value *originToIntptr(IRBuilder<> &IRB, Value *Origin) {
671 const DataLayout &DL = F.getParent()->getDataLayout();
672 unsigned IntptrSize = DL.getTypeStoreSize(MS.IntptrTy);
673 if (IntptrSize == kOriginSize) return Origin;
674 assert(IntptrSize == kOriginSize * 2);
675 Origin = IRB.CreateIntCast(Origin, MS.IntptrTy, /* isSigned */ false);
676 return IRB.CreateOr(Origin, IRB.CreateShl(Origin, kOriginSize * 8));
677 }
678
679 /// \brief Fill memory range with the given origin value.
paintOrigin__anon69a70b300211::MemorySanitizerVisitor680 void paintOrigin(IRBuilder<> &IRB, Value *Origin, Value *OriginPtr,
681 unsigned Size, unsigned Alignment) {
682 const DataLayout &DL = F.getParent()->getDataLayout();
683 unsigned IntptrAlignment = DL.getABITypeAlignment(MS.IntptrTy);
684 unsigned IntptrSize = DL.getTypeStoreSize(MS.IntptrTy);
685 assert(IntptrAlignment >= kMinOriginAlignment);
686 assert(IntptrSize >= kOriginSize);
687
688 unsigned Ofs = 0;
689 unsigned CurrentAlignment = Alignment;
690 if (Alignment >= IntptrAlignment && IntptrSize > kOriginSize) {
691 Value *IntptrOrigin = originToIntptr(IRB, Origin);
692 Value *IntptrOriginPtr =
693 IRB.CreatePointerCast(OriginPtr, PointerType::get(MS.IntptrTy, 0));
694 for (unsigned i = 0; i < Size / IntptrSize; ++i) {
695 Value *Ptr = i ? IRB.CreateConstGEP1_32(MS.IntptrTy, IntptrOriginPtr, i)
696 : IntptrOriginPtr;
697 IRB.CreateAlignedStore(IntptrOrigin, Ptr, CurrentAlignment);
698 Ofs += IntptrSize / kOriginSize;
699 CurrentAlignment = IntptrAlignment;
700 }
701 }
702
703 for (unsigned i = Ofs; i < (Size + kOriginSize - 1) / kOriginSize; ++i) {
704 Value *GEP =
705 i ? IRB.CreateConstGEP1_32(nullptr, OriginPtr, i) : OriginPtr;
706 IRB.CreateAlignedStore(Origin, GEP, CurrentAlignment);
707 CurrentAlignment = kMinOriginAlignment;
708 }
709 }
710
storeOrigin__anon69a70b300211::MemorySanitizerVisitor711 void storeOrigin(IRBuilder<> &IRB, Value *Addr, Value *Shadow, Value *Origin,
712 unsigned Alignment, bool AsCall) {
713 const DataLayout &DL = F.getParent()->getDataLayout();
714 unsigned OriginAlignment = std::max(kMinOriginAlignment, Alignment);
715 unsigned StoreSize = DL.getTypeStoreSize(Shadow->getType());
716 if (Shadow->getType()->isAggregateType()) {
717 paintOrigin(IRB, updateOrigin(Origin, IRB),
718 getOriginPtr(Addr, IRB, Alignment), StoreSize,
719 OriginAlignment);
720 } else {
721 Value *ConvertedShadow = convertToShadowTyNoVec(Shadow, IRB);
722 Constant *ConstantShadow = dyn_cast_or_null<Constant>(ConvertedShadow);
723 if (ConstantShadow) {
724 if (ClCheckConstantShadow && !ConstantShadow->isZeroValue())
725 paintOrigin(IRB, updateOrigin(Origin, IRB),
726 getOriginPtr(Addr, IRB, Alignment), StoreSize,
727 OriginAlignment);
728 return;
729 }
730
731 unsigned TypeSizeInBits =
732 DL.getTypeSizeInBits(ConvertedShadow->getType());
733 unsigned SizeIndex = TypeSizeToSizeIndex(TypeSizeInBits);
734 if (AsCall && SizeIndex < kNumberOfAccessSizes) {
735 Value *Fn = MS.MaybeStoreOriginFn[SizeIndex];
736 Value *ConvertedShadow2 = IRB.CreateZExt(
737 ConvertedShadow, IRB.getIntNTy(8 * (1 << SizeIndex)));
738 IRB.CreateCall(Fn, {ConvertedShadow2,
739 IRB.CreatePointerCast(Addr, IRB.getInt8PtrTy()),
740 Origin});
741 } else {
742 Value *Cmp = IRB.CreateICmpNE(
743 ConvertedShadow, getCleanShadow(ConvertedShadow), "_mscmp");
744 Instruction *CheckTerm = SplitBlockAndInsertIfThen(
745 Cmp, &*IRB.GetInsertPoint(), false, MS.OriginStoreWeights);
746 IRBuilder<> IRBNew(CheckTerm);
747 paintOrigin(IRBNew, updateOrigin(Origin, IRBNew),
748 getOriginPtr(Addr, IRBNew, Alignment), StoreSize,
749 OriginAlignment);
750 }
751 }
752 }
753
materializeStores__anon69a70b300211::MemorySanitizerVisitor754 void materializeStores(bool InstrumentWithCalls) {
755 for (StoreInst *SI : StoreList) {
756 IRBuilder<> IRB(SI);
757 Value *Val = SI->getValueOperand();
758 Value *Addr = SI->getPointerOperand();
759 Value *Shadow = SI->isAtomic() ? getCleanShadow(Val) : getShadow(Val);
760 Value *ShadowPtr = getShadowPtr(Addr, Shadow->getType(), IRB);
761
762 StoreInst *NewSI =
763 IRB.CreateAlignedStore(Shadow, ShadowPtr, SI->getAlignment());
764 DEBUG(dbgs() << " STORE: " << *NewSI << "\n");
765 (void)NewSI;
766
767 if (ClCheckAccessAddress)
768 insertShadowCheck(Addr, SI);
769
770 if (SI->isAtomic())
771 SI->setOrdering(addReleaseOrdering(SI->getOrdering()));
772
773 if (MS.TrackOrigins && !SI->isAtomic())
774 storeOrigin(IRB, Addr, Shadow, getOrigin(Val), SI->getAlignment(),
775 InstrumentWithCalls);
776 }
777 }
778
materializeOneCheck__anon69a70b300211::MemorySanitizerVisitor779 void materializeOneCheck(Instruction *OrigIns, Value *Shadow, Value *Origin,
780 bool AsCall) {
781 IRBuilder<> IRB(OrigIns);
782 DEBUG(dbgs() << " SHAD0 : " << *Shadow << "\n");
783 Value *ConvertedShadow = convertToShadowTyNoVec(Shadow, IRB);
784 DEBUG(dbgs() << " SHAD1 : " << *ConvertedShadow << "\n");
785
786 Constant *ConstantShadow = dyn_cast_or_null<Constant>(ConvertedShadow);
787 if (ConstantShadow) {
788 if (ClCheckConstantShadow && !ConstantShadow->isZeroValue()) {
789 if (MS.TrackOrigins) {
790 IRB.CreateStore(Origin ? (Value *)Origin : (Value *)IRB.getInt32(0),
791 MS.OriginTLS);
792 }
793 IRB.CreateCall(MS.WarningFn, {});
794 IRB.CreateCall(MS.EmptyAsm, {});
795 // FIXME: Insert UnreachableInst if !ClKeepGoing?
796 // This may invalidate some of the following checks and needs to be done
797 // at the very end.
798 }
799 return;
800 }
801
802 const DataLayout &DL = OrigIns->getModule()->getDataLayout();
803
804 unsigned TypeSizeInBits = DL.getTypeSizeInBits(ConvertedShadow->getType());
805 unsigned SizeIndex = TypeSizeToSizeIndex(TypeSizeInBits);
806 if (AsCall && SizeIndex < kNumberOfAccessSizes) {
807 Value *Fn = MS.MaybeWarningFn[SizeIndex];
808 Value *ConvertedShadow2 =
809 IRB.CreateZExt(ConvertedShadow, IRB.getIntNTy(8 * (1 << SizeIndex)));
810 IRB.CreateCall(Fn, {ConvertedShadow2, MS.TrackOrigins && Origin
811 ? Origin
812 : (Value *)IRB.getInt32(0)});
813 } else {
814 Value *Cmp = IRB.CreateICmpNE(ConvertedShadow,
815 getCleanShadow(ConvertedShadow), "_mscmp");
816 Instruction *CheckTerm = SplitBlockAndInsertIfThen(
817 Cmp, OrigIns,
818 /* Unreachable */ !ClKeepGoing, MS.ColdCallWeights);
819
820 IRB.SetInsertPoint(CheckTerm);
821 if (MS.TrackOrigins) {
822 IRB.CreateStore(Origin ? (Value *)Origin : (Value *)IRB.getInt32(0),
823 MS.OriginTLS);
824 }
825 IRB.CreateCall(MS.WarningFn, {});
826 IRB.CreateCall(MS.EmptyAsm, {});
827 DEBUG(dbgs() << " CHECK: " << *Cmp << "\n");
828 }
829 }
830
materializeChecks__anon69a70b300211::MemorySanitizerVisitor831 void materializeChecks(bool InstrumentWithCalls) {
832 for (const auto &ShadowData : InstrumentationList) {
833 Instruction *OrigIns = ShadowData.OrigIns;
834 Value *Shadow = ShadowData.Shadow;
835 Value *Origin = ShadowData.Origin;
836 materializeOneCheck(OrigIns, Shadow, Origin, InstrumentWithCalls);
837 }
838 DEBUG(dbgs() << "DONE:\n" << F);
839 }
840
841 /// \brief Add MemorySanitizer instrumentation to a function.
runOnFunction__anon69a70b300211::MemorySanitizerVisitor842 bool runOnFunction() {
843 MS.initializeCallbacks(*F.getParent());
844
845 // In the presence of unreachable blocks, we may see Phi nodes with
846 // incoming nodes from such blocks. Since InstVisitor skips unreachable
847 // blocks, such nodes will not have any shadow value associated with them.
848 // It's easier to remove unreachable blocks than deal with missing shadow.
849 removeUnreachableBlocks(F);
850
851 // Iterate all BBs in depth-first order and create shadow instructions
852 // for all instructions (where applicable).
853 // For PHI nodes we create dummy shadow PHIs which will be finalized later.
854 for (BasicBlock *BB : depth_first(&F.getEntryBlock()))
855 visit(*BB);
856
857
858 // Finalize PHI nodes.
859 for (PHINode *PN : ShadowPHINodes) {
860 PHINode *PNS = cast<PHINode>(getShadow(PN));
861 PHINode *PNO = MS.TrackOrigins ? cast<PHINode>(getOrigin(PN)) : nullptr;
862 size_t NumValues = PN->getNumIncomingValues();
863 for (size_t v = 0; v < NumValues; v++) {
864 PNS->addIncoming(getShadow(PN, v), PN->getIncomingBlock(v));
865 if (PNO) PNO->addIncoming(getOrigin(PN, v), PN->getIncomingBlock(v));
866 }
867 }
868
869 VAHelper->finalizeInstrumentation();
870
871 bool InstrumentWithCalls = ClInstrumentationWithCallThreshold >= 0 &&
872 InstrumentationList.size() + StoreList.size() >
873 (unsigned)ClInstrumentationWithCallThreshold;
874
875 // Delayed instrumentation of StoreInst.
876 // This may add new checks to be inserted later.
877 materializeStores(InstrumentWithCalls);
878
879 // Insert shadow value checks.
880 materializeChecks(InstrumentWithCalls);
881
882 return true;
883 }
884
885 /// \brief Compute the shadow type that corresponds to a given Value.
getShadowTy__anon69a70b300211::MemorySanitizerVisitor886 Type *getShadowTy(Value *V) {
887 return getShadowTy(V->getType());
888 }
889
890 /// \brief Compute the shadow type that corresponds to a given Type.
getShadowTy__anon69a70b300211::MemorySanitizerVisitor891 Type *getShadowTy(Type *OrigTy) {
892 if (!OrigTy->isSized()) {
893 return nullptr;
894 }
895 // For integer type, shadow is the same as the original type.
896 // This may return weird-sized types like i1.
897 if (IntegerType *IT = dyn_cast<IntegerType>(OrigTy))
898 return IT;
899 const DataLayout &DL = F.getParent()->getDataLayout();
900 if (VectorType *VT = dyn_cast<VectorType>(OrigTy)) {
901 uint32_t EltSize = DL.getTypeSizeInBits(VT->getElementType());
902 return VectorType::get(IntegerType::get(*MS.C, EltSize),
903 VT->getNumElements());
904 }
905 if (ArrayType *AT = dyn_cast<ArrayType>(OrigTy)) {
906 return ArrayType::get(getShadowTy(AT->getElementType()),
907 AT->getNumElements());
908 }
909 if (StructType *ST = dyn_cast<StructType>(OrigTy)) {
910 SmallVector<Type*, 4> Elements;
911 for (unsigned i = 0, n = ST->getNumElements(); i < n; i++)
912 Elements.push_back(getShadowTy(ST->getElementType(i)));
913 StructType *Res = StructType::get(*MS.C, Elements, ST->isPacked());
914 DEBUG(dbgs() << "getShadowTy: " << *ST << " ===> " << *Res << "\n");
915 return Res;
916 }
917 uint32_t TypeSize = DL.getTypeSizeInBits(OrigTy);
918 return IntegerType::get(*MS.C, TypeSize);
919 }
920
921 /// \brief Flatten a vector type.
getShadowTyNoVec__anon69a70b300211::MemorySanitizerVisitor922 Type *getShadowTyNoVec(Type *ty) {
923 if (VectorType *vt = dyn_cast<VectorType>(ty))
924 return IntegerType::get(*MS.C, vt->getBitWidth());
925 return ty;
926 }
927
928 /// \brief Convert a shadow value to it's flattened variant.
convertToShadowTyNoVec__anon69a70b300211::MemorySanitizerVisitor929 Value *convertToShadowTyNoVec(Value *V, IRBuilder<> &IRB) {
930 Type *Ty = V->getType();
931 Type *NoVecTy = getShadowTyNoVec(Ty);
932 if (Ty == NoVecTy) return V;
933 return IRB.CreateBitCast(V, NoVecTy);
934 }
935
936 /// \brief Compute the integer shadow offset that corresponds to a given
937 /// application address.
938 ///
939 /// Offset = (Addr & ~AndMask) ^ XorMask
getShadowPtrOffset__anon69a70b300211::MemorySanitizerVisitor940 Value *getShadowPtrOffset(Value *Addr, IRBuilder<> &IRB) {
941 Value *OffsetLong = IRB.CreatePointerCast(Addr, MS.IntptrTy);
942
943 uint64_t AndMask = MS.MapParams->AndMask;
944 if (AndMask)
945 OffsetLong =
946 IRB.CreateAnd(OffsetLong, ConstantInt::get(MS.IntptrTy, ~AndMask));
947
948 uint64_t XorMask = MS.MapParams->XorMask;
949 if (XorMask)
950 OffsetLong =
951 IRB.CreateXor(OffsetLong, ConstantInt::get(MS.IntptrTy, XorMask));
952 return OffsetLong;
953 }
954
955 /// \brief Compute the shadow address that corresponds to a given application
956 /// address.
957 ///
958 /// Shadow = ShadowBase + Offset
getShadowPtr__anon69a70b300211::MemorySanitizerVisitor959 Value *getShadowPtr(Value *Addr, Type *ShadowTy,
960 IRBuilder<> &IRB) {
961 Value *ShadowLong = getShadowPtrOffset(Addr, IRB);
962 uint64_t ShadowBase = MS.MapParams->ShadowBase;
963 if (ShadowBase != 0)
964 ShadowLong =
965 IRB.CreateAdd(ShadowLong,
966 ConstantInt::get(MS.IntptrTy, ShadowBase));
967 return IRB.CreateIntToPtr(ShadowLong, PointerType::get(ShadowTy, 0));
968 }
969
970 /// \brief Compute the origin address that corresponds to a given application
971 /// address.
972 ///
973 /// OriginAddr = (OriginBase + Offset) & ~3ULL
getOriginPtr__anon69a70b300211::MemorySanitizerVisitor974 Value *getOriginPtr(Value *Addr, IRBuilder<> &IRB, unsigned Alignment) {
975 Value *OriginLong = getShadowPtrOffset(Addr, IRB);
976 uint64_t OriginBase = MS.MapParams->OriginBase;
977 if (OriginBase != 0)
978 OriginLong =
979 IRB.CreateAdd(OriginLong,
980 ConstantInt::get(MS.IntptrTy, OriginBase));
981 if (Alignment < kMinOriginAlignment) {
982 uint64_t Mask = kMinOriginAlignment - 1;
983 OriginLong = IRB.CreateAnd(OriginLong,
984 ConstantInt::get(MS.IntptrTy, ~Mask));
985 }
986 return IRB.CreateIntToPtr(OriginLong,
987 PointerType::get(IRB.getInt32Ty(), 0));
988 }
989
990 /// \brief Compute the shadow address for a given function argument.
991 ///
992 /// Shadow = ParamTLS+ArgOffset.
getShadowPtrForArgument__anon69a70b300211::MemorySanitizerVisitor993 Value *getShadowPtrForArgument(Value *A, IRBuilder<> &IRB,
994 int ArgOffset) {
995 Value *Base = IRB.CreatePointerCast(MS.ParamTLS, MS.IntptrTy);
996 Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
997 return IRB.CreateIntToPtr(Base, PointerType::get(getShadowTy(A), 0),
998 "_msarg");
999 }
1000
1001 /// \brief Compute the origin address for a given function argument.
getOriginPtrForArgument__anon69a70b300211::MemorySanitizerVisitor1002 Value *getOriginPtrForArgument(Value *A, IRBuilder<> &IRB,
1003 int ArgOffset) {
1004 if (!MS.TrackOrigins) return nullptr;
1005 Value *Base = IRB.CreatePointerCast(MS.ParamOriginTLS, MS.IntptrTy);
1006 Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
1007 return IRB.CreateIntToPtr(Base, PointerType::get(MS.OriginTy, 0),
1008 "_msarg_o");
1009 }
1010
1011 /// \brief Compute the shadow address for a retval.
getShadowPtrForRetval__anon69a70b300211::MemorySanitizerVisitor1012 Value *getShadowPtrForRetval(Value *A, IRBuilder<> &IRB) {
1013 Value *Base = IRB.CreatePointerCast(MS.RetvalTLS, MS.IntptrTy);
1014 return IRB.CreateIntToPtr(Base, PointerType::get(getShadowTy(A), 0),
1015 "_msret");
1016 }
1017
1018 /// \brief Compute the origin address for a retval.
getOriginPtrForRetval__anon69a70b300211::MemorySanitizerVisitor1019 Value *getOriginPtrForRetval(IRBuilder<> &IRB) {
1020 // We keep a single origin for the entire retval. Might be too optimistic.
1021 return MS.RetvalOriginTLS;
1022 }
1023
1024 /// \brief Set SV to be the shadow value for V.
setShadow__anon69a70b300211::MemorySanitizerVisitor1025 void setShadow(Value *V, Value *SV) {
1026 assert(!ShadowMap.count(V) && "Values may only have one shadow");
1027 ShadowMap[V] = PropagateShadow ? SV : getCleanShadow(V);
1028 }
1029
1030 /// \brief Set Origin to be the origin value for V.
setOrigin__anon69a70b300211::MemorySanitizerVisitor1031 void setOrigin(Value *V, Value *Origin) {
1032 if (!MS.TrackOrigins) return;
1033 assert(!OriginMap.count(V) && "Values may only have one origin");
1034 DEBUG(dbgs() << "ORIGIN: " << *V << " ==> " << *Origin << "\n");
1035 OriginMap[V] = Origin;
1036 }
1037
1038 /// \brief Create a clean shadow value for a given value.
1039 ///
1040 /// Clean shadow (all zeroes) means all bits of the value are defined
1041 /// (initialized).
getCleanShadow__anon69a70b300211::MemorySanitizerVisitor1042 Constant *getCleanShadow(Value *V) {
1043 Type *ShadowTy = getShadowTy(V);
1044 if (!ShadowTy)
1045 return nullptr;
1046 return Constant::getNullValue(ShadowTy);
1047 }
1048
1049 /// \brief Create a dirty shadow of a given shadow type.
getPoisonedShadow__anon69a70b300211::MemorySanitizerVisitor1050 Constant *getPoisonedShadow(Type *ShadowTy) {
1051 assert(ShadowTy);
1052 if (isa<IntegerType>(ShadowTy) || isa<VectorType>(ShadowTy))
1053 return Constant::getAllOnesValue(ShadowTy);
1054 if (ArrayType *AT = dyn_cast<ArrayType>(ShadowTy)) {
1055 SmallVector<Constant *, 4> Vals(AT->getNumElements(),
1056 getPoisonedShadow(AT->getElementType()));
1057 return ConstantArray::get(AT, Vals);
1058 }
1059 if (StructType *ST = dyn_cast<StructType>(ShadowTy)) {
1060 SmallVector<Constant *, 4> Vals;
1061 for (unsigned i = 0, n = ST->getNumElements(); i < n; i++)
1062 Vals.push_back(getPoisonedShadow(ST->getElementType(i)));
1063 return ConstantStruct::get(ST, Vals);
1064 }
1065 llvm_unreachable("Unexpected shadow type");
1066 }
1067
1068 /// \brief Create a dirty shadow for a given value.
getPoisonedShadow__anon69a70b300211::MemorySanitizerVisitor1069 Constant *getPoisonedShadow(Value *V) {
1070 Type *ShadowTy = getShadowTy(V);
1071 if (!ShadowTy)
1072 return nullptr;
1073 return getPoisonedShadow(ShadowTy);
1074 }
1075
1076 /// \brief Create a clean (zero) origin.
getCleanOrigin__anon69a70b300211::MemorySanitizerVisitor1077 Value *getCleanOrigin() {
1078 return Constant::getNullValue(MS.OriginTy);
1079 }
1080
1081 /// \brief Get the shadow value for a given Value.
1082 ///
1083 /// This function either returns the value set earlier with setShadow,
1084 /// or extracts if from ParamTLS (for function arguments).
getShadow__anon69a70b300211::MemorySanitizerVisitor1085 Value *getShadow(Value *V) {
1086 if (!PropagateShadow) return getCleanShadow(V);
1087 if (Instruction *I = dyn_cast<Instruction>(V)) {
1088 // For instructions the shadow is already stored in the map.
1089 Value *Shadow = ShadowMap[V];
1090 if (!Shadow) {
1091 DEBUG(dbgs() << "No shadow: " << *V << "\n" << *(I->getParent()));
1092 (void)I;
1093 assert(Shadow && "No shadow for a value");
1094 }
1095 return Shadow;
1096 }
1097 if (UndefValue *U = dyn_cast<UndefValue>(V)) {
1098 Value *AllOnes = PoisonUndef ? getPoisonedShadow(V) : getCleanShadow(V);
1099 DEBUG(dbgs() << "Undef: " << *U << " ==> " << *AllOnes << "\n");
1100 (void)U;
1101 return AllOnes;
1102 }
1103 if (Argument *A = dyn_cast<Argument>(V)) {
1104 // For arguments we compute the shadow on demand and store it in the map.
1105 Value **ShadowPtr = &ShadowMap[V];
1106 if (*ShadowPtr)
1107 return *ShadowPtr;
1108 Function *F = A->getParent();
1109 IRBuilder<> EntryIRB(F->getEntryBlock().getFirstNonPHI());
1110 unsigned ArgOffset = 0;
1111 const DataLayout &DL = F->getParent()->getDataLayout();
1112 for (auto &FArg : F->args()) {
1113 if (!FArg.getType()->isSized()) {
1114 DEBUG(dbgs() << "Arg is not sized\n");
1115 continue;
1116 }
1117 unsigned Size =
1118 FArg.hasByValAttr()
1119 ? DL.getTypeAllocSize(FArg.getType()->getPointerElementType())
1120 : DL.getTypeAllocSize(FArg.getType());
1121 if (A == &FArg) {
1122 bool Overflow = ArgOffset + Size > kParamTLSSize;
1123 Value *Base = getShadowPtrForArgument(&FArg, EntryIRB, ArgOffset);
1124 if (FArg.hasByValAttr()) {
1125 // ByVal pointer itself has clean shadow. We copy the actual
1126 // argument shadow to the underlying memory.
1127 // Figure out maximal valid memcpy alignment.
1128 unsigned ArgAlign = FArg.getParamAlignment();
1129 if (ArgAlign == 0) {
1130 Type *EltType = A->getType()->getPointerElementType();
1131 ArgAlign = DL.getABITypeAlignment(EltType);
1132 }
1133 if (Overflow) {
1134 // ParamTLS overflow.
1135 EntryIRB.CreateMemSet(
1136 getShadowPtr(V, EntryIRB.getInt8Ty(), EntryIRB),
1137 Constant::getNullValue(EntryIRB.getInt8Ty()), Size, ArgAlign);
1138 } else {
1139 unsigned CopyAlign = std::min(ArgAlign, kShadowTLSAlignment);
1140 Value *Cpy = EntryIRB.CreateMemCpy(
1141 getShadowPtr(V, EntryIRB.getInt8Ty(), EntryIRB), Base, Size,
1142 CopyAlign);
1143 DEBUG(dbgs() << " ByValCpy: " << *Cpy << "\n");
1144 (void)Cpy;
1145 }
1146 *ShadowPtr = getCleanShadow(V);
1147 } else {
1148 if (Overflow) {
1149 // ParamTLS overflow.
1150 *ShadowPtr = getCleanShadow(V);
1151 } else {
1152 *ShadowPtr =
1153 EntryIRB.CreateAlignedLoad(Base, kShadowTLSAlignment);
1154 }
1155 }
1156 DEBUG(dbgs() << " ARG: " << FArg << " ==> " <<
1157 **ShadowPtr << "\n");
1158 if (MS.TrackOrigins && !Overflow) {
1159 Value *OriginPtr =
1160 getOriginPtrForArgument(&FArg, EntryIRB, ArgOffset);
1161 setOrigin(A, EntryIRB.CreateLoad(OriginPtr));
1162 } else {
1163 setOrigin(A, getCleanOrigin());
1164 }
1165 }
1166 ArgOffset += alignTo(Size, kShadowTLSAlignment);
1167 }
1168 assert(*ShadowPtr && "Could not find shadow for an argument");
1169 return *ShadowPtr;
1170 }
1171 // For everything else the shadow is zero.
1172 return getCleanShadow(V);
1173 }
1174
1175 /// \brief Get the shadow for i-th argument of the instruction I.
getShadow__anon69a70b300211::MemorySanitizerVisitor1176 Value *getShadow(Instruction *I, int i) {
1177 return getShadow(I->getOperand(i));
1178 }
1179
1180 /// \brief Get the origin for a value.
getOrigin__anon69a70b300211::MemorySanitizerVisitor1181 Value *getOrigin(Value *V) {
1182 if (!MS.TrackOrigins) return nullptr;
1183 if (!PropagateShadow) return getCleanOrigin();
1184 if (isa<Constant>(V)) return getCleanOrigin();
1185 assert((isa<Instruction>(V) || isa<Argument>(V)) &&
1186 "Unexpected value type in getOrigin()");
1187 Value *Origin = OriginMap[V];
1188 assert(Origin && "Missing origin");
1189 return Origin;
1190 }
1191
1192 /// \brief Get the origin for i-th argument of the instruction I.
getOrigin__anon69a70b300211::MemorySanitizerVisitor1193 Value *getOrigin(Instruction *I, int i) {
1194 return getOrigin(I->getOperand(i));
1195 }
1196
1197 /// \brief Remember the place where a shadow check should be inserted.
1198 ///
1199 /// This location will be later instrumented with a check that will print a
1200 /// UMR warning in runtime if the shadow value is not 0.
insertShadowCheck__anon69a70b300211::MemorySanitizerVisitor1201 void insertShadowCheck(Value *Shadow, Value *Origin, Instruction *OrigIns) {
1202 assert(Shadow);
1203 if (!InsertChecks) return;
1204 #ifndef NDEBUG
1205 Type *ShadowTy = Shadow->getType();
1206 assert((isa<IntegerType>(ShadowTy) || isa<VectorType>(ShadowTy)) &&
1207 "Can only insert checks for integer and vector shadow types");
1208 #endif
1209 InstrumentationList.push_back(
1210 ShadowOriginAndInsertPoint(Shadow, Origin, OrigIns));
1211 }
1212
1213 /// \brief Remember the place where a shadow check should be inserted.
1214 ///
1215 /// This location will be later instrumented with a check that will print a
1216 /// UMR warning in runtime if the value is not fully defined.
insertShadowCheck__anon69a70b300211::MemorySanitizerVisitor1217 void insertShadowCheck(Value *Val, Instruction *OrigIns) {
1218 assert(Val);
1219 Value *Shadow, *Origin;
1220 if (ClCheckConstantShadow) {
1221 Shadow = getShadow(Val);
1222 if (!Shadow) return;
1223 Origin = getOrigin(Val);
1224 } else {
1225 Shadow = dyn_cast_or_null<Instruction>(getShadow(Val));
1226 if (!Shadow) return;
1227 Origin = dyn_cast_or_null<Instruction>(getOrigin(Val));
1228 }
1229 insertShadowCheck(Shadow, Origin, OrigIns);
1230 }
1231
addReleaseOrdering__anon69a70b300211::MemorySanitizerVisitor1232 AtomicOrdering addReleaseOrdering(AtomicOrdering a) {
1233 switch (a) {
1234 case AtomicOrdering::NotAtomic:
1235 return AtomicOrdering::NotAtomic;
1236 case AtomicOrdering::Unordered:
1237 case AtomicOrdering::Monotonic:
1238 case AtomicOrdering::Release:
1239 return AtomicOrdering::Release;
1240 case AtomicOrdering::Acquire:
1241 case AtomicOrdering::AcquireRelease:
1242 return AtomicOrdering::AcquireRelease;
1243 case AtomicOrdering::SequentiallyConsistent:
1244 return AtomicOrdering::SequentiallyConsistent;
1245 }
1246 llvm_unreachable("Unknown ordering");
1247 }
1248
addAcquireOrdering__anon69a70b300211::MemorySanitizerVisitor1249 AtomicOrdering addAcquireOrdering(AtomicOrdering a) {
1250 switch (a) {
1251 case AtomicOrdering::NotAtomic:
1252 return AtomicOrdering::NotAtomic;
1253 case AtomicOrdering::Unordered:
1254 case AtomicOrdering::Monotonic:
1255 case AtomicOrdering::Acquire:
1256 return AtomicOrdering::Acquire;
1257 case AtomicOrdering::Release:
1258 case AtomicOrdering::AcquireRelease:
1259 return AtomicOrdering::AcquireRelease;
1260 case AtomicOrdering::SequentiallyConsistent:
1261 return AtomicOrdering::SequentiallyConsistent;
1262 }
1263 llvm_unreachable("Unknown ordering");
1264 }
1265
1266 // ------------------- Visitors.
1267
1268 /// \brief Instrument LoadInst
1269 ///
1270 /// Loads the corresponding shadow and (optionally) origin.
1271 /// Optionally, checks that the load address is fully defined.
visitLoadInst__anon69a70b300211::MemorySanitizerVisitor1272 void visitLoadInst(LoadInst &I) {
1273 assert(I.getType()->isSized() && "Load type must have size");
1274 IRBuilder<> IRB(I.getNextNode());
1275 Type *ShadowTy = getShadowTy(&I);
1276 Value *Addr = I.getPointerOperand();
1277 if (PropagateShadow && !I.getMetadata("nosanitize")) {
1278 Value *ShadowPtr = getShadowPtr(Addr, ShadowTy, IRB);
1279 setShadow(&I,
1280 IRB.CreateAlignedLoad(ShadowPtr, I.getAlignment(), "_msld"));
1281 } else {
1282 setShadow(&I, getCleanShadow(&I));
1283 }
1284
1285 if (ClCheckAccessAddress)
1286 insertShadowCheck(I.getPointerOperand(), &I);
1287
1288 if (I.isAtomic())
1289 I.setOrdering(addAcquireOrdering(I.getOrdering()));
1290
1291 if (MS.TrackOrigins) {
1292 if (PropagateShadow) {
1293 unsigned Alignment = I.getAlignment();
1294 unsigned OriginAlignment = std::max(kMinOriginAlignment, Alignment);
1295 setOrigin(&I, IRB.CreateAlignedLoad(getOriginPtr(Addr, IRB, Alignment),
1296 OriginAlignment));
1297 } else {
1298 setOrigin(&I, getCleanOrigin());
1299 }
1300 }
1301 }
1302
1303 /// \brief Instrument StoreInst
1304 ///
1305 /// Stores the corresponding shadow and (optionally) origin.
1306 /// Optionally, checks that the store address is fully defined.
visitStoreInst__anon69a70b300211::MemorySanitizerVisitor1307 void visitStoreInst(StoreInst &I) {
1308 StoreList.push_back(&I);
1309 }
1310
handleCASOrRMW__anon69a70b300211::MemorySanitizerVisitor1311 void handleCASOrRMW(Instruction &I) {
1312 assert(isa<AtomicRMWInst>(I) || isa<AtomicCmpXchgInst>(I));
1313
1314 IRBuilder<> IRB(&I);
1315 Value *Addr = I.getOperand(0);
1316 Value *ShadowPtr = getShadowPtr(Addr, I.getType(), IRB);
1317
1318 if (ClCheckAccessAddress)
1319 insertShadowCheck(Addr, &I);
1320
1321 // Only test the conditional argument of cmpxchg instruction.
1322 // The other argument can potentially be uninitialized, but we can not
1323 // detect this situation reliably without possible false positives.
1324 if (isa<AtomicCmpXchgInst>(I))
1325 insertShadowCheck(I.getOperand(1), &I);
1326
1327 IRB.CreateStore(getCleanShadow(&I), ShadowPtr);
1328
1329 setShadow(&I, getCleanShadow(&I));
1330 setOrigin(&I, getCleanOrigin());
1331 }
1332
visitAtomicRMWInst__anon69a70b300211::MemorySanitizerVisitor1333 void visitAtomicRMWInst(AtomicRMWInst &I) {
1334 handleCASOrRMW(I);
1335 I.setOrdering(addReleaseOrdering(I.getOrdering()));
1336 }
1337
visitAtomicCmpXchgInst__anon69a70b300211::MemorySanitizerVisitor1338 void visitAtomicCmpXchgInst(AtomicCmpXchgInst &I) {
1339 handleCASOrRMW(I);
1340 I.setSuccessOrdering(addReleaseOrdering(I.getSuccessOrdering()));
1341 }
1342
1343 // Vector manipulation.
visitExtractElementInst__anon69a70b300211::MemorySanitizerVisitor1344 void visitExtractElementInst(ExtractElementInst &I) {
1345 insertShadowCheck(I.getOperand(1), &I);
1346 IRBuilder<> IRB(&I);
1347 setShadow(&I, IRB.CreateExtractElement(getShadow(&I, 0), I.getOperand(1),
1348 "_msprop"));
1349 setOrigin(&I, getOrigin(&I, 0));
1350 }
1351
visitInsertElementInst__anon69a70b300211::MemorySanitizerVisitor1352 void visitInsertElementInst(InsertElementInst &I) {
1353 insertShadowCheck(I.getOperand(2), &I);
1354 IRBuilder<> IRB(&I);
1355 setShadow(&I, IRB.CreateInsertElement(getShadow(&I, 0), getShadow(&I, 1),
1356 I.getOperand(2), "_msprop"));
1357 setOriginForNaryOp(I);
1358 }
1359
visitShuffleVectorInst__anon69a70b300211::MemorySanitizerVisitor1360 void visitShuffleVectorInst(ShuffleVectorInst &I) {
1361 insertShadowCheck(I.getOperand(2), &I);
1362 IRBuilder<> IRB(&I);
1363 setShadow(&I, IRB.CreateShuffleVector(getShadow(&I, 0), getShadow(&I, 1),
1364 I.getOperand(2), "_msprop"));
1365 setOriginForNaryOp(I);
1366 }
1367
1368 // Casts.
visitSExtInst__anon69a70b300211::MemorySanitizerVisitor1369 void visitSExtInst(SExtInst &I) {
1370 IRBuilder<> IRB(&I);
1371 setShadow(&I, IRB.CreateSExt(getShadow(&I, 0), I.getType(), "_msprop"));
1372 setOrigin(&I, getOrigin(&I, 0));
1373 }
1374
visitZExtInst__anon69a70b300211::MemorySanitizerVisitor1375 void visitZExtInst(ZExtInst &I) {
1376 IRBuilder<> IRB(&I);
1377 setShadow(&I, IRB.CreateZExt(getShadow(&I, 0), I.getType(), "_msprop"));
1378 setOrigin(&I, getOrigin(&I, 0));
1379 }
1380
visitTruncInst__anon69a70b300211::MemorySanitizerVisitor1381 void visitTruncInst(TruncInst &I) {
1382 IRBuilder<> IRB(&I);
1383 setShadow(&I, IRB.CreateTrunc(getShadow(&I, 0), I.getType(), "_msprop"));
1384 setOrigin(&I, getOrigin(&I, 0));
1385 }
1386
visitBitCastInst__anon69a70b300211::MemorySanitizerVisitor1387 void visitBitCastInst(BitCastInst &I) {
1388 // Special case: if this is the bitcast (there is exactly 1 allowed) between
1389 // a musttail call and a ret, don't instrument. New instructions are not
1390 // allowed after a musttail call.
1391 if (auto *CI = dyn_cast<CallInst>(I.getOperand(0)))
1392 if (CI->isMustTailCall())
1393 return;
1394 IRBuilder<> IRB(&I);
1395 setShadow(&I, IRB.CreateBitCast(getShadow(&I, 0), getShadowTy(&I)));
1396 setOrigin(&I, getOrigin(&I, 0));
1397 }
1398
visitPtrToIntInst__anon69a70b300211::MemorySanitizerVisitor1399 void visitPtrToIntInst(PtrToIntInst &I) {
1400 IRBuilder<> IRB(&I);
1401 setShadow(&I, IRB.CreateIntCast(getShadow(&I, 0), getShadowTy(&I), false,
1402 "_msprop_ptrtoint"));
1403 setOrigin(&I, getOrigin(&I, 0));
1404 }
1405
visitIntToPtrInst__anon69a70b300211::MemorySanitizerVisitor1406 void visitIntToPtrInst(IntToPtrInst &I) {
1407 IRBuilder<> IRB(&I);
1408 setShadow(&I, IRB.CreateIntCast(getShadow(&I, 0), getShadowTy(&I), false,
1409 "_msprop_inttoptr"));
1410 setOrigin(&I, getOrigin(&I, 0));
1411 }
1412
visitFPToSIInst__anon69a70b300211::MemorySanitizerVisitor1413 void visitFPToSIInst(CastInst& I) { handleShadowOr(I); }
visitFPToUIInst__anon69a70b300211::MemorySanitizerVisitor1414 void visitFPToUIInst(CastInst& I) { handleShadowOr(I); }
visitSIToFPInst__anon69a70b300211::MemorySanitizerVisitor1415 void visitSIToFPInst(CastInst& I) { handleShadowOr(I); }
visitUIToFPInst__anon69a70b300211::MemorySanitizerVisitor1416 void visitUIToFPInst(CastInst& I) { handleShadowOr(I); }
visitFPExtInst__anon69a70b300211::MemorySanitizerVisitor1417 void visitFPExtInst(CastInst& I) { handleShadowOr(I); }
visitFPTruncInst__anon69a70b300211::MemorySanitizerVisitor1418 void visitFPTruncInst(CastInst& I) { handleShadowOr(I); }
1419
1420 /// \brief Propagate shadow for bitwise AND.
1421 ///
1422 /// This code is exact, i.e. if, for example, a bit in the left argument
1423 /// is defined and 0, then neither the value not definedness of the
1424 /// corresponding bit in B don't affect the resulting shadow.
visitAnd__anon69a70b300211::MemorySanitizerVisitor1425 void visitAnd(BinaryOperator &I) {
1426 IRBuilder<> IRB(&I);
1427 // "And" of 0 and a poisoned value results in unpoisoned value.
1428 // 1&1 => 1; 0&1 => 0; p&1 => p;
1429 // 1&0 => 0; 0&0 => 0; p&0 => 0;
1430 // 1&p => p; 0&p => 0; p&p => p;
1431 // S = (S1 & S2) | (V1 & S2) | (S1 & V2)
1432 Value *S1 = getShadow(&I, 0);
1433 Value *S2 = getShadow(&I, 1);
1434 Value *V1 = I.getOperand(0);
1435 Value *V2 = I.getOperand(1);
1436 if (V1->getType() != S1->getType()) {
1437 V1 = IRB.CreateIntCast(V1, S1->getType(), false);
1438 V2 = IRB.CreateIntCast(V2, S2->getType(), false);
1439 }
1440 Value *S1S2 = IRB.CreateAnd(S1, S2);
1441 Value *V1S2 = IRB.CreateAnd(V1, S2);
1442 Value *S1V2 = IRB.CreateAnd(S1, V2);
1443 setShadow(&I, IRB.CreateOr(S1S2, IRB.CreateOr(V1S2, S1V2)));
1444 setOriginForNaryOp(I);
1445 }
1446
visitOr__anon69a70b300211::MemorySanitizerVisitor1447 void visitOr(BinaryOperator &I) {
1448 IRBuilder<> IRB(&I);
1449 // "Or" of 1 and a poisoned value results in unpoisoned value.
1450 // 1|1 => 1; 0|1 => 1; p|1 => 1;
1451 // 1|0 => 1; 0|0 => 0; p|0 => p;
1452 // 1|p => 1; 0|p => p; p|p => p;
1453 // S = (S1 & S2) | (~V1 & S2) | (S1 & ~V2)
1454 Value *S1 = getShadow(&I, 0);
1455 Value *S2 = getShadow(&I, 1);
1456 Value *V1 = IRB.CreateNot(I.getOperand(0));
1457 Value *V2 = IRB.CreateNot(I.getOperand(1));
1458 if (V1->getType() != S1->getType()) {
1459 V1 = IRB.CreateIntCast(V1, S1->getType(), false);
1460 V2 = IRB.CreateIntCast(V2, S2->getType(), false);
1461 }
1462 Value *S1S2 = IRB.CreateAnd(S1, S2);
1463 Value *V1S2 = IRB.CreateAnd(V1, S2);
1464 Value *S1V2 = IRB.CreateAnd(S1, V2);
1465 setShadow(&I, IRB.CreateOr(S1S2, IRB.CreateOr(V1S2, S1V2)));
1466 setOriginForNaryOp(I);
1467 }
1468
1469 /// \brief Default propagation of shadow and/or origin.
1470 ///
1471 /// This class implements the general case of shadow propagation, used in all
1472 /// cases where we don't know and/or don't care about what the operation
1473 /// actually does. It converts all input shadow values to a common type
1474 /// (extending or truncating as necessary), and bitwise OR's them.
1475 ///
1476 /// This is much cheaper than inserting checks (i.e. requiring inputs to be
1477 /// fully initialized), and less prone to false positives.
1478 ///
1479 /// This class also implements the general case of origin propagation. For a
1480 /// Nary operation, result origin is set to the origin of an argument that is
1481 /// not entirely initialized. If there is more than one such arguments, the
1482 /// rightmost of them is picked. It does not matter which one is picked if all
1483 /// arguments are initialized.
1484 template <bool CombineShadow>
1485 class Combiner {
1486 Value *Shadow;
1487 Value *Origin;
1488 IRBuilder<> &IRB;
1489 MemorySanitizerVisitor *MSV;
1490
1491 public:
Combiner(MemorySanitizerVisitor * MSV,IRBuilder<> & IRB)1492 Combiner(MemorySanitizerVisitor *MSV, IRBuilder<> &IRB) :
1493 Shadow(nullptr), Origin(nullptr), IRB(IRB), MSV(MSV) {}
1494
1495 /// \brief Add a pair of shadow and origin values to the mix.
Add(Value * OpShadow,Value * OpOrigin)1496 Combiner &Add(Value *OpShadow, Value *OpOrigin) {
1497 if (CombineShadow) {
1498 assert(OpShadow);
1499 if (!Shadow)
1500 Shadow = OpShadow;
1501 else {
1502 OpShadow = MSV->CreateShadowCast(IRB, OpShadow, Shadow->getType());
1503 Shadow = IRB.CreateOr(Shadow, OpShadow, "_msprop");
1504 }
1505 }
1506
1507 if (MSV->MS.TrackOrigins) {
1508 assert(OpOrigin);
1509 if (!Origin) {
1510 Origin = OpOrigin;
1511 } else {
1512 Constant *ConstOrigin = dyn_cast<Constant>(OpOrigin);
1513 // No point in adding something that might result in 0 origin value.
1514 if (!ConstOrigin || !ConstOrigin->isNullValue()) {
1515 Value *FlatShadow = MSV->convertToShadowTyNoVec(OpShadow, IRB);
1516 Value *Cond =
1517 IRB.CreateICmpNE(FlatShadow, MSV->getCleanShadow(FlatShadow));
1518 Origin = IRB.CreateSelect(Cond, OpOrigin, Origin);
1519 }
1520 }
1521 }
1522 return *this;
1523 }
1524
1525 /// \brief Add an application value to the mix.
Add(Value * V)1526 Combiner &Add(Value *V) {
1527 Value *OpShadow = MSV->getShadow(V);
1528 Value *OpOrigin = MSV->MS.TrackOrigins ? MSV->getOrigin(V) : nullptr;
1529 return Add(OpShadow, OpOrigin);
1530 }
1531
1532 /// \brief Set the current combined values as the given instruction's shadow
1533 /// and origin.
Done(Instruction * I)1534 void Done(Instruction *I) {
1535 if (CombineShadow) {
1536 assert(Shadow);
1537 Shadow = MSV->CreateShadowCast(IRB, Shadow, MSV->getShadowTy(I));
1538 MSV->setShadow(I, Shadow);
1539 }
1540 if (MSV->MS.TrackOrigins) {
1541 assert(Origin);
1542 MSV->setOrigin(I, Origin);
1543 }
1544 }
1545 };
1546
1547 typedef Combiner<true> ShadowAndOriginCombiner;
1548 typedef Combiner<false> OriginCombiner;
1549
1550 /// \brief Propagate origin for arbitrary operation.
setOriginForNaryOp__anon69a70b300211::MemorySanitizerVisitor1551 void setOriginForNaryOp(Instruction &I) {
1552 if (!MS.TrackOrigins) return;
1553 IRBuilder<> IRB(&I);
1554 OriginCombiner OC(this, IRB);
1555 for (Instruction::op_iterator OI = I.op_begin(); OI != I.op_end(); ++OI)
1556 OC.Add(OI->get());
1557 OC.Done(&I);
1558 }
1559
VectorOrPrimitiveTypeSizeInBits__anon69a70b300211::MemorySanitizerVisitor1560 size_t VectorOrPrimitiveTypeSizeInBits(Type *Ty) {
1561 assert(!(Ty->isVectorTy() && Ty->getScalarType()->isPointerTy()) &&
1562 "Vector of pointers is not a valid shadow type");
1563 return Ty->isVectorTy() ?
1564 Ty->getVectorNumElements() * Ty->getScalarSizeInBits() :
1565 Ty->getPrimitiveSizeInBits();
1566 }
1567
1568 /// \brief Cast between two shadow types, extending or truncating as
1569 /// necessary.
CreateShadowCast__anon69a70b300211::MemorySanitizerVisitor1570 Value *CreateShadowCast(IRBuilder<> &IRB, Value *V, Type *dstTy,
1571 bool Signed = false) {
1572 Type *srcTy = V->getType();
1573 if (dstTy->isIntegerTy() && srcTy->isIntegerTy())
1574 return IRB.CreateIntCast(V, dstTy, Signed);
1575 if (dstTy->isVectorTy() && srcTy->isVectorTy() &&
1576 dstTy->getVectorNumElements() == srcTy->getVectorNumElements())
1577 return IRB.CreateIntCast(V, dstTy, Signed);
1578 size_t srcSizeInBits = VectorOrPrimitiveTypeSizeInBits(srcTy);
1579 size_t dstSizeInBits = VectorOrPrimitiveTypeSizeInBits(dstTy);
1580 Value *V1 = IRB.CreateBitCast(V, Type::getIntNTy(*MS.C, srcSizeInBits));
1581 Value *V2 =
1582 IRB.CreateIntCast(V1, Type::getIntNTy(*MS.C, dstSizeInBits), Signed);
1583 return IRB.CreateBitCast(V2, dstTy);
1584 // TODO: handle struct types.
1585 }
1586
1587 /// \brief Cast an application value to the type of its own shadow.
CreateAppToShadowCast__anon69a70b300211::MemorySanitizerVisitor1588 Value *CreateAppToShadowCast(IRBuilder<> &IRB, Value *V) {
1589 Type *ShadowTy = getShadowTy(V);
1590 if (V->getType() == ShadowTy)
1591 return V;
1592 if (V->getType()->isPtrOrPtrVectorTy())
1593 return IRB.CreatePtrToInt(V, ShadowTy);
1594 else
1595 return IRB.CreateBitCast(V, ShadowTy);
1596 }
1597
1598 /// \brief Propagate shadow for arbitrary operation.
handleShadowOr__anon69a70b300211::MemorySanitizerVisitor1599 void handleShadowOr(Instruction &I) {
1600 IRBuilder<> IRB(&I);
1601 ShadowAndOriginCombiner SC(this, IRB);
1602 for (Instruction::op_iterator OI = I.op_begin(); OI != I.op_end(); ++OI)
1603 SC.Add(OI->get());
1604 SC.Done(&I);
1605 }
1606
1607 // \brief Handle multiplication by constant.
1608 //
1609 // Handle a special case of multiplication by constant that may have one or
1610 // more zeros in the lower bits. This makes corresponding number of lower bits
1611 // of the result zero as well. We model it by shifting the other operand
1612 // shadow left by the required number of bits. Effectively, we transform
1613 // (X * (A * 2**B)) to ((X << B) * A) and instrument (X << B) as (Sx << B).
1614 // We use multiplication by 2**N instead of shift to cover the case of
1615 // multiplication by 0, which may occur in some elements of a vector operand.
handleMulByConstant__anon69a70b300211::MemorySanitizerVisitor1616 void handleMulByConstant(BinaryOperator &I, Constant *ConstArg,
1617 Value *OtherArg) {
1618 Constant *ShadowMul;
1619 Type *Ty = ConstArg->getType();
1620 if (Ty->isVectorTy()) {
1621 unsigned NumElements = Ty->getVectorNumElements();
1622 Type *EltTy = Ty->getSequentialElementType();
1623 SmallVector<Constant *, 16> Elements;
1624 for (unsigned Idx = 0; Idx < NumElements; ++Idx) {
1625 if (ConstantInt *Elt =
1626 dyn_cast<ConstantInt>(ConstArg->getAggregateElement(Idx))) {
1627 const APInt &V = Elt->getValue();
1628 APInt V2 = APInt(V.getBitWidth(), 1) << V.countTrailingZeros();
1629 Elements.push_back(ConstantInt::get(EltTy, V2));
1630 } else {
1631 Elements.push_back(ConstantInt::get(EltTy, 1));
1632 }
1633 }
1634 ShadowMul = ConstantVector::get(Elements);
1635 } else {
1636 if (ConstantInt *Elt = dyn_cast<ConstantInt>(ConstArg)) {
1637 const APInt &V = Elt->getValue();
1638 APInt V2 = APInt(V.getBitWidth(), 1) << V.countTrailingZeros();
1639 ShadowMul = ConstantInt::get(Ty, V2);
1640 } else {
1641 ShadowMul = ConstantInt::get(Ty, 1);
1642 }
1643 }
1644
1645 IRBuilder<> IRB(&I);
1646 setShadow(&I,
1647 IRB.CreateMul(getShadow(OtherArg), ShadowMul, "msprop_mul_cst"));
1648 setOrigin(&I, getOrigin(OtherArg));
1649 }
1650
visitMul__anon69a70b300211::MemorySanitizerVisitor1651 void visitMul(BinaryOperator &I) {
1652 Constant *constOp0 = dyn_cast<Constant>(I.getOperand(0));
1653 Constant *constOp1 = dyn_cast<Constant>(I.getOperand(1));
1654 if (constOp0 && !constOp1)
1655 handleMulByConstant(I, constOp0, I.getOperand(1));
1656 else if (constOp1 && !constOp0)
1657 handleMulByConstant(I, constOp1, I.getOperand(0));
1658 else
1659 handleShadowOr(I);
1660 }
1661
visitFAdd__anon69a70b300211::MemorySanitizerVisitor1662 void visitFAdd(BinaryOperator &I) { handleShadowOr(I); }
visitFSub__anon69a70b300211::MemorySanitizerVisitor1663 void visitFSub(BinaryOperator &I) { handleShadowOr(I); }
visitFMul__anon69a70b300211::MemorySanitizerVisitor1664 void visitFMul(BinaryOperator &I) { handleShadowOr(I); }
visitAdd__anon69a70b300211::MemorySanitizerVisitor1665 void visitAdd(BinaryOperator &I) { handleShadowOr(I); }
visitSub__anon69a70b300211::MemorySanitizerVisitor1666 void visitSub(BinaryOperator &I) { handleShadowOr(I); }
visitXor__anon69a70b300211::MemorySanitizerVisitor1667 void visitXor(BinaryOperator &I) { handleShadowOr(I); }
1668
handleDiv__anon69a70b300211::MemorySanitizerVisitor1669 void handleDiv(Instruction &I) {
1670 IRBuilder<> IRB(&I);
1671 // Strict on the second argument.
1672 insertShadowCheck(I.getOperand(1), &I);
1673 setShadow(&I, getShadow(&I, 0));
1674 setOrigin(&I, getOrigin(&I, 0));
1675 }
1676
visitUDiv__anon69a70b300211::MemorySanitizerVisitor1677 void visitUDiv(BinaryOperator &I) { handleDiv(I); }
visitSDiv__anon69a70b300211::MemorySanitizerVisitor1678 void visitSDiv(BinaryOperator &I) { handleDiv(I); }
visitFDiv__anon69a70b300211::MemorySanitizerVisitor1679 void visitFDiv(BinaryOperator &I) { handleDiv(I); }
visitURem__anon69a70b300211::MemorySanitizerVisitor1680 void visitURem(BinaryOperator &I) { handleDiv(I); }
visitSRem__anon69a70b300211::MemorySanitizerVisitor1681 void visitSRem(BinaryOperator &I) { handleDiv(I); }
visitFRem__anon69a70b300211::MemorySanitizerVisitor1682 void visitFRem(BinaryOperator &I) { handleDiv(I); }
1683
1684 /// \brief Instrument == and != comparisons.
1685 ///
1686 /// Sometimes the comparison result is known even if some of the bits of the
1687 /// arguments are not.
handleEqualityComparison__anon69a70b300211::MemorySanitizerVisitor1688 void handleEqualityComparison(ICmpInst &I) {
1689 IRBuilder<> IRB(&I);
1690 Value *A = I.getOperand(0);
1691 Value *B = I.getOperand(1);
1692 Value *Sa = getShadow(A);
1693 Value *Sb = getShadow(B);
1694
1695 // Get rid of pointers and vectors of pointers.
1696 // For ints (and vectors of ints), types of A and Sa match,
1697 // and this is a no-op.
1698 A = IRB.CreatePointerCast(A, Sa->getType());
1699 B = IRB.CreatePointerCast(B, Sb->getType());
1700
1701 // A == B <==> (C = A^B) == 0
1702 // A != B <==> (C = A^B) != 0
1703 // Sc = Sa | Sb
1704 Value *C = IRB.CreateXor(A, B);
1705 Value *Sc = IRB.CreateOr(Sa, Sb);
1706 // Now dealing with i = (C == 0) comparison (or C != 0, does not matter now)
1707 // Result is defined if one of the following is true
1708 // * there is a defined 1 bit in C
1709 // * C is fully defined
1710 // Si = !(C & ~Sc) && Sc
1711 Value *Zero = Constant::getNullValue(Sc->getType());
1712 Value *MinusOne = Constant::getAllOnesValue(Sc->getType());
1713 Value *Si =
1714 IRB.CreateAnd(IRB.CreateICmpNE(Sc, Zero),
1715 IRB.CreateICmpEQ(
1716 IRB.CreateAnd(IRB.CreateXor(Sc, MinusOne), C), Zero));
1717 Si->setName("_msprop_icmp");
1718 setShadow(&I, Si);
1719 setOriginForNaryOp(I);
1720 }
1721
1722 /// \brief Build the lowest possible value of V, taking into account V's
1723 /// uninitialized bits.
getLowestPossibleValue__anon69a70b300211::MemorySanitizerVisitor1724 Value *getLowestPossibleValue(IRBuilder<> &IRB, Value *A, Value *Sa,
1725 bool isSigned) {
1726 if (isSigned) {
1727 // Split shadow into sign bit and other bits.
1728 Value *SaOtherBits = IRB.CreateLShr(IRB.CreateShl(Sa, 1), 1);
1729 Value *SaSignBit = IRB.CreateXor(Sa, SaOtherBits);
1730 // Maximise the undefined shadow bit, minimize other undefined bits.
1731 return
1732 IRB.CreateOr(IRB.CreateAnd(A, IRB.CreateNot(SaOtherBits)), SaSignBit);
1733 } else {
1734 // Minimize undefined bits.
1735 return IRB.CreateAnd(A, IRB.CreateNot(Sa));
1736 }
1737 }
1738
1739 /// \brief Build the highest possible value of V, taking into account V's
1740 /// uninitialized bits.
getHighestPossibleValue__anon69a70b300211::MemorySanitizerVisitor1741 Value *getHighestPossibleValue(IRBuilder<> &IRB, Value *A, Value *Sa,
1742 bool isSigned) {
1743 if (isSigned) {
1744 // Split shadow into sign bit and other bits.
1745 Value *SaOtherBits = IRB.CreateLShr(IRB.CreateShl(Sa, 1), 1);
1746 Value *SaSignBit = IRB.CreateXor(Sa, SaOtherBits);
1747 // Minimise the undefined shadow bit, maximise other undefined bits.
1748 return
1749 IRB.CreateOr(IRB.CreateAnd(A, IRB.CreateNot(SaSignBit)), SaOtherBits);
1750 } else {
1751 // Maximize undefined bits.
1752 return IRB.CreateOr(A, Sa);
1753 }
1754 }
1755
1756 /// \brief Instrument relational comparisons.
1757 ///
1758 /// This function does exact shadow propagation for all relational
1759 /// comparisons of integers, pointers and vectors of those.
1760 /// FIXME: output seems suboptimal when one of the operands is a constant
handleRelationalComparisonExact__anon69a70b300211::MemorySanitizerVisitor1761 void handleRelationalComparisonExact(ICmpInst &I) {
1762 IRBuilder<> IRB(&I);
1763 Value *A = I.getOperand(0);
1764 Value *B = I.getOperand(1);
1765 Value *Sa = getShadow(A);
1766 Value *Sb = getShadow(B);
1767
1768 // Get rid of pointers and vectors of pointers.
1769 // For ints (and vectors of ints), types of A and Sa match,
1770 // and this is a no-op.
1771 A = IRB.CreatePointerCast(A, Sa->getType());
1772 B = IRB.CreatePointerCast(B, Sb->getType());
1773
1774 // Let [a0, a1] be the interval of possible values of A, taking into account
1775 // its undefined bits. Let [b0, b1] be the interval of possible values of B.
1776 // Then (A cmp B) is defined iff (a0 cmp b1) == (a1 cmp b0).
1777 bool IsSigned = I.isSigned();
1778 Value *S1 = IRB.CreateICmp(I.getPredicate(),
1779 getLowestPossibleValue(IRB, A, Sa, IsSigned),
1780 getHighestPossibleValue(IRB, B, Sb, IsSigned));
1781 Value *S2 = IRB.CreateICmp(I.getPredicate(),
1782 getHighestPossibleValue(IRB, A, Sa, IsSigned),
1783 getLowestPossibleValue(IRB, B, Sb, IsSigned));
1784 Value *Si = IRB.CreateXor(S1, S2);
1785 setShadow(&I, Si);
1786 setOriginForNaryOp(I);
1787 }
1788
1789 /// \brief Instrument signed relational comparisons.
1790 ///
1791 /// Handle sign bit tests: x<0, x>=0, x<=-1, x>-1 by propagating the highest
1792 /// bit of the shadow. Everything else is delegated to handleShadowOr().
handleSignedRelationalComparison__anon69a70b300211::MemorySanitizerVisitor1793 void handleSignedRelationalComparison(ICmpInst &I) {
1794 Constant *constOp;
1795 Value *op = nullptr;
1796 CmpInst::Predicate pre;
1797 if ((constOp = dyn_cast<Constant>(I.getOperand(1)))) {
1798 op = I.getOperand(0);
1799 pre = I.getPredicate();
1800 } else if ((constOp = dyn_cast<Constant>(I.getOperand(0)))) {
1801 op = I.getOperand(1);
1802 pre = I.getSwappedPredicate();
1803 } else {
1804 handleShadowOr(I);
1805 return;
1806 }
1807
1808 if ((constOp->isNullValue() &&
1809 (pre == CmpInst::ICMP_SLT || pre == CmpInst::ICMP_SGE)) ||
1810 (constOp->isAllOnesValue() &&
1811 (pre == CmpInst::ICMP_SGT || pre == CmpInst::ICMP_SLE))) {
1812 IRBuilder<> IRB(&I);
1813 Value *Shadow = IRB.CreateICmpSLT(getShadow(op), getCleanShadow(op),
1814 "_msprop_icmp_s");
1815 setShadow(&I, Shadow);
1816 setOrigin(&I, getOrigin(op));
1817 } else {
1818 handleShadowOr(I);
1819 }
1820 }
1821
visitICmpInst__anon69a70b300211::MemorySanitizerVisitor1822 void visitICmpInst(ICmpInst &I) {
1823 if (!ClHandleICmp) {
1824 handleShadowOr(I);
1825 return;
1826 }
1827 if (I.isEquality()) {
1828 handleEqualityComparison(I);
1829 return;
1830 }
1831
1832 assert(I.isRelational());
1833 if (ClHandleICmpExact) {
1834 handleRelationalComparisonExact(I);
1835 return;
1836 }
1837 if (I.isSigned()) {
1838 handleSignedRelationalComparison(I);
1839 return;
1840 }
1841
1842 assert(I.isUnsigned());
1843 if ((isa<Constant>(I.getOperand(0)) || isa<Constant>(I.getOperand(1)))) {
1844 handleRelationalComparisonExact(I);
1845 return;
1846 }
1847
1848 handleShadowOr(I);
1849 }
1850
visitFCmpInst__anon69a70b300211::MemorySanitizerVisitor1851 void visitFCmpInst(FCmpInst &I) {
1852 handleShadowOr(I);
1853 }
1854
handleShift__anon69a70b300211::MemorySanitizerVisitor1855 void handleShift(BinaryOperator &I) {
1856 IRBuilder<> IRB(&I);
1857 // If any of the S2 bits are poisoned, the whole thing is poisoned.
1858 // Otherwise perform the same shift on S1.
1859 Value *S1 = getShadow(&I, 0);
1860 Value *S2 = getShadow(&I, 1);
1861 Value *S2Conv = IRB.CreateSExt(IRB.CreateICmpNE(S2, getCleanShadow(S2)),
1862 S2->getType());
1863 Value *V2 = I.getOperand(1);
1864 Value *Shift = IRB.CreateBinOp(I.getOpcode(), S1, V2);
1865 setShadow(&I, IRB.CreateOr(Shift, S2Conv));
1866 setOriginForNaryOp(I);
1867 }
1868
visitShl__anon69a70b300211::MemorySanitizerVisitor1869 void visitShl(BinaryOperator &I) { handleShift(I); }
visitAShr__anon69a70b300211::MemorySanitizerVisitor1870 void visitAShr(BinaryOperator &I) { handleShift(I); }
visitLShr__anon69a70b300211::MemorySanitizerVisitor1871 void visitLShr(BinaryOperator &I) { handleShift(I); }
1872
1873 /// \brief Instrument llvm.memmove
1874 ///
1875 /// At this point we don't know if llvm.memmove will be inlined or not.
1876 /// If we don't instrument it and it gets inlined,
1877 /// our interceptor will not kick in and we will lose the memmove.
1878 /// If we instrument the call here, but it does not get inlined,
1879 /// we will memove the shadow twice: which is bad in case
1880 /// of overlapping regions. So, we simply lower the intrinsic to a call.
1881 ///
1882 /// Similar situation exists for memcpy and memset.
visitMemMoveInst__anon69a70b300211::MemorySanitizerVisitor1883 void visitMemMoveInst(MemMoveInst &I) {
1884 IRBuilder<> IRB(&I);
1885 IRB.CreateCall(
1886 MS.MemmoveFn,
1887 {IRB.CreatePointerCast(I.getArgOperand(0), IRB.getInt8PtrTy()),
1888 IRB.CreatePointerCast(I.getArgOperand(1), IRB.getInt8PtrTy()),
1889 IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)});
1890 I.eraseFromParent();
1891 }
1892
1893 // Similar to memmove: avoid copying shadow twice.
1894 // This is somewhat unfortunate as it may slowdown small constant memcpys.
1895 // FIXME: consider doing manual inline for small constant sizes and proper
1896 // alignment.
visitMemCpyInst__anon69a70b300211::MemorySanitizerVisitor1897 void visitMemCpyInst(MemCpyInst &I) {
1898 IRBuilder<> IRB(&I);
1899 IRB.CreateCall(
1900 MS.MemcpyFn,
1901 {IRB.CreatePointerCast(I.getArgOperand(0), IRB.getInt8PtrTy()),
1902 IRB.CreatePointerCast(I.getArgOperand(1), IRB.getInt8PtrTy()),
1903 IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)});
1904 I.eraseFromParent();
1905 }
1906
1907 // Same as memcpy.
visitMemSetInst__anon69a70b300211::MemorySanitizerVisitor1908 void visitMemSetInst(MemSetInst &I) {
1909 IRBuilder<> IRB(&I);
1910 IRB.CreateCall(
1911 MS.MemsetFn,
1912 {IRB.CreatePointerCast(I.getArgOperand(0), IRB.getInt8PtrTy()),
1913 IRB.CreateIntCast(I.getArgOperand(1), IRB.getInt32Ty(), false),
1914 IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)});
1915 I.eraseFromParent();
1916 }
1917
visitVAStartInst__anon69a70b300211::MemorySanitizerVisitor1918 void visitVAStartInst(VAStartInst &I) {
1919 VAHelper->visitVAStartInst(I);
1920 }
1921
visitVACopyInst__anon69a70b300211::MemorySanitizerVisitor1922 void visitVACopyInst(VACopyInst &I) {
1923 VAHelper->visitVACopyInst(I);
1924 }
1925
1926 /// \brief Handle vector store-like intrinsics.
1927 ///
1928 /// Instrument intrinsics that look like a simple SIMD store: writes memory,
1929 /// has 1 pointer argument and 1 vector argument, returns void.
handleVectorStoreIntrinsic__anon69a70b300211::MemorySanitizerVisitor1930 bool handleVectorStoreIntrinsic(IntrinsicInst &I) {
1931 IRBuilder<> IRB(&I);
1932 Value* Addr = I.getArgOperand(0);
1933 Value *Shadow = getShadow(&I, 1);
1934 Value *ShadowPtr = getShadowPtr(Addr, Shadow->getType(), IRB);
1935
1936 // We don't know the pointer alignment (could be unaligned SSE store!).
1937 // Have to assume to worst case.
1938 IRB.CreateAlignedStore(Shadow, ShadowPtr, 1);
1939
1940 if (ClCheckAccessAddress)
1941 insertShadowCheck(Addr, &I);
1942
1943 // FIXME: use ClStoreCleanOrigin
1944 // FIXME: factor out common code from materializeStores
1945 if (MS.TrackOrigins)
1946 IRB.CreateStore(getOrigin(&I, 1), getOriginPtr(Addr, IRB, 1));
1947 return true;
1948 }
1949
1950 /// \brief Handle vector load-like intrinsics.
1951 ///
1952 /// Instrument intrinsics that look like a simple SIMD load: reads memory,
1953 /// has 1 pointer argument, returns a vector.
handleVectorLoadIntrinsic__anon69a70b300211::MemorySanitizerVisitor1954 bool handleVectorLoadIntrinsic(IntrinsicInst &I) {
1955 IRBuilder<> IRB(&I);
1956 Value *Addr = I.getArgOperand(0);
1957
1958 Type *ShadowTy = getShadowTy(&I);
1959 if (PropagateShadow) {
1960 Value *ShadowPtr = getShadowPtr(Addr, ShadowTy, IRB);
1961 // We don't know the pointer alignment (could be unaligned SSE load!).
1962 // Have to assume to worst case.
1963 setShadow(&I, IRB.CreateAlignedLoad(ShadowPtr, 1, "_msld"));
1964 } else {
1965 setShadow(&I, getCleanShadow(&I));
1966 }
1967
1968 if (ClCheckAccessAddress)
1969 insertShadowCheck(Addr, &I);
1970
1971 if (MS.TrackOrigins) {
1972 if (PropagateShadow)
1973 setOrigin(&I, IRB.CreateLoad(getOriginPtr(Addr, IRB, 1)));
1974 else
1975 setOrigin(&I, getCleanOrigin());
1976 }
1977 return true;
1978 }
1979
1980 /// \brief Handle (SIMD arithmetic)-like intrinsics.
1981 ///
1982 /// Instrument intrinsics with any number of arguments of the same type,
1983 /// equal to the return type. The type should be simple (no aggregates or
1984 /// pointers; vectors are fine).
1985 /// Caller guarantees that this intrinsic does not access memory.
maybeHandleSimpleNomemIntrinsic__anon69a70b300211::MemorySanitizerVisitor1986 bool maybeHandleSimpleNomemIntrinsic(IntrinsicInst &I) {
1987 Type *RetTy = I.getType();
1988 if (!(RetTy->isIntOrIntVectorTy() ||
1989 RetTy->isFPOrFPVectorTy() ||
1990 RetTy->isX86_MMXTy()))
1991 return false;
1992
1993 unsigned NumArgOperands = I.getNumArgOperands();
1994
1995 for (unsigned i = 0; i < NumArgOperands; ++i) {
1996 Type *Ty = I.getArgOperand(i)->getType();
1997 if (Ty != RetTy)
1998 return false;
1999 }
2000
2001 IRBuilder<> IRB(&I);
2002 ShadowAndOriginCombiner SC(this, IRB);
2003 for (unsigned i = 0; i < NumArgOperands; ++i)
2004 SC.Add(I.getArgOperand(i));
2005 SC.Done(&I);
2006
2007 return true;
2008 }
2009
2010 /// \brief Heuristically instrument unknown intrinsics.
2011 ///
2012 /// The main purpose of this code is to do something reasonable with all
2013 /// random intrinsics we might encounter, most importantly - SIMD intrinsics.
2014 /// We recognize several classes of intrinsics by their argument types and
2015 /// ModRefBehaviour and apply special intrumentation when we are reasonably
2016 /// sure that we know what the intrinsic does.
2017 ///
2018 /// We special-case intrinsics where this approach fails. See llvm.bswap
2019 /// handling as an example of that.
handleUnknownIntrinsic__anon69a70b300211::MemorySanitizerVisitor2020 bool handleUnknownIntrinsic(IntrinsicInst &I) {
2021 unsigned NumArgOperands = I.getNumArgOperands();
2022 if (NumArgOperands == 0)
2023 return false;
2024
2025 if (NumArgOperands == 2 &&
2026 I.getArgOperand(0)->getType()->isPointerTy() &&
2027 I.getArgOperand(1)->getType()->isVectorTy() &&
2028 I.getType()->isVoidTy() &&
2029 !I.onlyReadsMemory()) {
2030 // This looks like a vector store.
2031 return handleVectorStoreIntrinsic(I);
2032 }
2033
2034 if (NumArgOperands == 1 &&
2035 I.getArgOperand(0)->getType()->isPointerTy() &&
2036 I.getType()->isVectorTy() &&
2037 I.onlyReadsMemory()) {
2038 // This looks like a vector load.
2039 return handleVectorLoadIntrinsic(I);
2040 }
2041
2042 if (I.doesNotAccessMemory())
2043 if (maybeHandleSimpleNomemIntrinsic(I))
2044 return true;
2045
2046 // FIXME: detect and handle SSE maskstore/maskload
2047 return false;
2048 }
2049
handleBswap__anon69a70b300211::MemorySanitizerVisitor2050 void handleBswap(IntrinsicInst &I) {
2051 IRBuilder<> IRB(&I);
2052 Value *Op = I.getArgOperand(0);
2053 Type *OpType = Op->getType();
2054 Function *BswapFunc = Intrinsic::getDeclaration(
2055 F.getParent(), Intrinsic::bswap, makeArrayRef(&OpType, 1));
2056 setShadow(&I, IRB.CreateCall(BswapFunc, getShadow(Op)));
2057 setOrigin(&I, getOrigin(Op));
2058 }
2059
2060 // \brief Instrument vector convert instrinsic.
2061 //
2062 // This function instruments intrinsics like cvtsi2ss:
2063 // %Out = int_xxx_cvtyyy(%ConvertOp)
2064 // or
2065 // %Out = int_xxx_cvtyyy(%CopyOp, %ConvertOp)
2066 // Intrinsic converts \p NumUsedElements elements of \p ConvertOp to the same
2067 // number \p Out elements, and (if has 2 arguments) copies the rest of the
2068 // elements from \p CopyOp.
2069 // In most cases conversion involves floating-point value which may trigger a
2070 // hardware exception when not fully initialized. For this reason we require
2071 // \p ConvertOp[0:NumUsedElements] to be fully initialized and trap otherwise.
2072 // We copy the shadow of \p CopyOp[NumUsedElements:] to \p
2073 // Out[NumUsedElements:]. This means that intrinsics without \p CopyOp always
2074 // return a fully initialized value.
handleVectorConvertIntrinsic__anon69a70b300211::MemorySanitizerVisitor2075 void handleVectorConvertIntrinsic(IntrinsicInst &I, int NumUsedElements) {
2076 IRBuilder<> IRB(&I);
2077 Value *CopyOp, *ConvertOp;
2078
2079 switch (I.getNumArgOperands()) {
2080 case 3:
2081 assert(isa<ConstantInt>(I.getArgOperand(2)) && "Invalid rounding mode");
2082 case 2:
2083 CopyOp = I.getArgOperand(0);
2084 ConvertOp = I.getArgOperand(1);
2085 break;
2086 case 1:
2087 ConvertOp = I.getArgOperand(0);
2088 CopyOp = nullptr;
2089 break;
2090 default:
2091 llvm_unreachable("Cvt intrinsic with unsupported number of arguments.");
2092 }
2093
2094 // The first *NumUsedElements* elements of ConvertOp are converted to the
2095 // same number of output elements. The rest of the output is copied from
2096 // CopyOp, or (if not available) filled with zeroes.
2097 // Combine shadow for elements of ConvertOp that are used in this operation,
2098 // and insert a check.
2099 // FIXME: consider propagating shadow of ConvertOp, at least in the case of
2100 // int->any conversion.
2101 Value *ConvertShadow = getShadow(ConvertOp);
2102 Value *AggShadow = nullptr;
2103 if (ConvertOp->getType()->isVectorTy()) {
2104 AggShadow = IRB.CreateExtractElement(
2105 ConvertShadow, ConstantInt::get(IRB.getInt32Ty(), 0));
2106 for (int i = 1; i < NumUsedElements; ++i) {
2107 Value *MoreShadow = IRB.CreateExtractElement(
2108 ConvertShadow, ConstantInt::get(IRB.getInt32Ty(), i));
2109 AggShadow = IRB.CreateOr(AggShadow, MoreShadow);
2110 }
2111 } else {
2112 AggShadow = ConvertShadow;
2113 }
2114 assert(AggShadow->getType()->isIntegerTy());
2115 insertShadowCheck(AggShadow, getOrigin(ConvertOp), &I);
2116
2117 // Build result shadow by zero-filling parts of CopyOp shadow that come from
2118 // ConvertOp.
2119 if (CopyOp) {
2120 assert(CopyOp->getType() == I.getType());
2121 assert(CopyOp->getType()->isVectorTy());
2122 Value *ResultShadow = getShadow(CopyOp);
2123 Type *EltTy = ResultShadow->getType()->getVectorElementType();
2124 for (int i = 0; i < NumUsedElements; ++i) {
2125 ResultShadow = IRB.CreateInsertElement(
2126 ResultShadow, ConstantInt::getNullValue(EltTy),
2127 ConstantInt::get(IRB.getInt32Ty(), i));
2128 }
2129 setShadow(&I, ResultShadow);
2130 setOrigin(&I, getOrigin(CopyOp));
2131 } else {
2132 setShadow(&I, getCleanShadow(&I));
2133 setOrigin(&I, getCleanOrigin());
2134 }
2135 }
2136
2137 // Given a scalar or vector, extract lower 64 bits (or less), and return all
2138 // zeroes if it is zero, and all ones otherwise.
Lower64ShadowExtend__anon69a70b300211::MemorySanitizerVisitor2139 Value *Lower64ShadowExtend(IRBuilder<> &IRB, Value *S, Type *T) {
2140 if (S->getType()->isVectorTy())
2141 S = CreateShadowCast(IRB, S, IRB.getInt64Ty(), /* Signed */ true);
2142 assert(S->getType()->getPrimitiveSizeInBits() <= 64);
2143 Value *S2 = IRB.CreateICmpNE(S, getCleanShadow(S));
2144 return CreateShadowCast(IRB, S2, T, /* Signed */ true);
2145 }
2146
2147 // Given a vector, extract its first element, and return all
2148 // zeroes if it is zero, and all ones otherwise.
LowerElementShadowExtend__anon69a70b300211::MemorySanitizerVisitor2149 Value *LowerElementShadowExtend(IRBuilder<> &IRB, Value *S, Type *T) {
2150 Value *S1 = IRB.CreateExtractElement(S, (uint64_t)0);
2151 Value *S2 = IRB.CreateICmpNE(S1, getCleanShadow(S1));
2152 return CreateShadowCast(IRB, S2, T, /* Signed */ true);
2153 }
2154
VariableShadowExtend__anon69a70b300211::MemorySanitizerVisitor2155 Value *VariableShadowExtend(IRBuilder<> &IRB, Value *S) {
2156 Type *T = S->getType();
2157 assert(T->isVectorTy());
2158 Value *S2 = IRB.CreateICmpNE(S, getCleanShadow(S));
2159 return IRB.CreateSExt(S2, T);
2160 }
2161
2162 // \brief Instrument vector shift instrinsic.
2163 //
2164 // This function instruments intrinsics like int_x86_avx2_psll_w.
2165 // Intrinsic shifts %In by %ShiftSize bits.
2166 // %ShiftSize may be a vector. In that case the lower 64 bits determine shift
2167 // size, and the rest is ignored. Behavior is defined even if shift size is
2168 // greater than register (or field) width.
handleVectorShiftIntrinsic__anon69a70b300211::MemorySanitizerVisitor2169 void handleVectorShiftIntrinsic(IntrinsicInst &I, bool Variable) {
2170 assert(I.getNumArgOperands() == 2);
2171 IRBuilder<> IRB(&I);
2172 // If any of the S2 bits are poisoned, the whole thing is poisoned.
2173 // Otherwise perform the same shift on S1.
2174 Value *S1 = getShadow(&I, 0);
2175 Value *S2 = getShadow(&I, 1);
2176 Value *S2Conv = Variable ? VariableShadowExtend(IRB, S2)
2177 : Lower64ShadowExtend(IRB, S2, getShadowTy(&I));
2178 Value *V1 = I.getOperand(0);
2179 Value *V2 = I.getOperand(1);
2180 Value *Shift = IRB.CreateCall(I.getCalledValue(),
2181 {IRB.CreateBitCast(S1, V1->getType()), V2});
2182 Shift = IRB.CreateBitCast(Shift, getShadowTy(&I));
2183 setShadow(&I, IRB.CreateOr(Shift, S2Conv));
2184 setOriginForNaryOp(I);
2185 }
2186
2187 // \brief Get an X86_MMX-sized vector type.
getMMXVectorTy__anon69a70b300211::MemorySanitizerVisitor2188 Type *getMMXVectorTy(unsigned EltSizeInBits) {
2189 const unsigned X86_MMXSizeInBits = 64;
2190 return VectorType::get(IntegerType::get(*MS.C, EltSizeInBits),
2191 X86_MMXSizeInBits / EltSizeInBits);
2192 }
2193
2194 // \brief Returns a signed counterpart for an (un)signed-saturate-and-pack
2195 // intrinsic.
getSignedPackIntrinsic__anon69a70b300211::MemorySanitizerVisitor2196 Intrinsic::ID getSignedPackIntrinsic(Intrinsic::ID id) {
2197 switch (id) {
2198 case llvm::Intrinsic::x86_sse2_packsswb_128:
2199 case llvm::Intrinsic::x86_sse2_packuswb_128:
2200 return llvm::Intrinsic::x86_sse2_packsswb_128;
2201
2202 case llvm::Intrinsic::x86_sse2_packssdw_128:
2203 case llvm::Intrinsic::x86_sse41_packusdw:
2204 return llvm::Intrinsic::x86_sse2_packssdw_128;
2205
2206 case llvm::Intrinsic::x86_avx2_packsswb:
2207 case llvm::Intrinsic::x86_avx2_packuswb:
2208 return llvm::Intrinsic::x86_avx2_packsswb;
2209
2210 case llvm::Intrinsic::x86_avx2_packssdw:
2211 case llvm::Intrinsic::x86_avx2_packusdw:
2212 return llvm::Intrinsic::x86_avx2_packssdw;
2213
2214 case llvm::Intrinsic::x86_mmx_packsswb:
2215 case llvm::Intrinsic::x86_mmx_packuswb:
2216 return llvm::Intrinsic::x86_mmx_packsswb;
2217
2218 case llvm::Intrinsic::x86_mmx_packssdw:
2219 return llvm::Intrinsic::x86_mmx_packssdw;
2220 default:
2221 llvm_unreachable("unexpected intrinsic id");
2222 }
2223 }
2224
2225 // \brief Instrument vector pack instrinsic.
2226 //
2227 // This function instruments intrinsics like x86_mmx_packsswb, that
2228 // packs elements of 2 input vectors into half as many bits with saturation.
2229 // Shadow is propagated with the signed variant of the same intrinsic applied
2230 // to sext(Sa != zeroinitializer), sext(Sb != zeroinitializer).
2231 // EltSizeInBits is used only for x86mmx arguments.
handleVectorPackIntrinsic__anon69a70b300211::MemorySanitizerVisitor2232 void handleVectorPackIntrinsic(IntrinsicInst &I, unsigned EltSizeInBits = 0) {
2233 assert(I.getNumArgOperands() == 2);
2234 bool isX86_MMX = I.getOperand(0)->getType()->isX86_MMXTy();
2235 IRBuilder<> IRB(&I);
2236 Value *S1 = getShadow(&I, 0);
2237 Value *S2 = getShadow(&I, 1);
2238 assert(isX86_MMX || S1->getType()->isVectorTy());
2239
2240 // SExt and ICmpNE below must apply to individual elements of input vectors.
2241 // In case of x86mmx arguments, cast them to appropriate vector types and
2242 // back.
2243 Type *T = isX86_MMX ? getMMXVectorTy(EltSizeInBits) : S1->getType();
2244 if (isX86_MMX) {
2245 S1 = IRB.CreateBitCast(S1, T);
2246 S2 = IRB.CreateBitCast(S2, T);
2247 }
2248 Value *S1_ext = IRB.CreateSExt(
2249 IRB.CreateICmpNE(S1, llvm::Constant::getNullValue(T)), T);
2250 Value *S2_ext = IRB.CreateSExt(
2251 IRB.CreateICmpNE(S2, llvm::Constant::getNullValue(T)), T);
2252 if (isX86_MMX) {
2253 Type *X86_MMXTy = Type::getX86_MMXTy(*MS.C);
2254 S1_ext = IRB.CreateBitCast(S1_ext, X86_MMXTy);
2255 S2_ext = IRB.CreateBitCast(S2_ext, X86_MMXTy);
2256 }
2257
2258 Function *ShadowFn = Intrinsic::getDeclaration(
2259 F.getParent(), getSignedPackIntrinsic(I.getIntrinsicID()));
2260
2261 Value *S =
2262 IRB.CreateCall(ShadowFn, {S1_ext, S2_ext}, "_msprop_vector_pack");
2263 if (isX86_MMX) S = IRB.CreateBitCast(S, getShadowTy(&I));
2264 setShadow(&I, S);
2265 setOriginForNaryOp(I);
2266 }
2267
2268 // \brief Instrument sum-of-absolute-differencies intrinsic.
handleVectorSadIntrinsic__anon69a70b300211::MemorySanitizerVisitor2269 void handleVectorSadIntrinsic(IntrinsicInst &I) {
2270 const unsigned SignificantBitsPerResultElement = 16;
2271 bool isX86_MMX = I.getOperand(0)->getType()->isX86_MMXTy();
2272 Type *ResTy = isX86_MMX ? IntegerType::get(*MS.C, 64) : I.getType();
2273 unsigned ZeroBitsPerResultElement =
2274 ResTy->getScalarSizeInBits() - SignificantBitsPerResultElement;
2275
2276 IRBuilder<> IRB(&I);
2277 Value *S = IRB.CreateOr(getShadow(&I, 0), getShadow(&I, 1));
2278 S = IRB.CreateBitCast(S, ResTy);
2279 S = IRB.CreateSExt(IRB.CreateICmpNE(S, Constant::getNullValue(ResTy)),
2280 ResTy);
2281 S = IRB.CreateLShr(S, ZeroBitsPerResultElement);
2282 S = IRB.CreateBitCast(S, getShadowTy(&I));
2283 setShadow(&I, S);
2284 setOriginForNaryOp(I);
2285 }
2286
2287 // \brief Instrument multiply-add intrinsic.
handleVectorPmaddIntrinsic__anon69a70b300211::MemorySanitizerVisitor2288 void handleVectorPmaddIntrinsic(IntrinsicInst &I,
2289 unsigned EltSizeInBits = 0) {
2290 bool isX86_MMX = I.getOperand(0)->getType()->isX86_MMXTy();
2291 Type *ResTy = isX86_MMX ? getMMXVectorTy(EltSizeInBits * 2) : I.getType();
2292 IRBuilder<> IRB(&I);
2293 Value *S = IRB.CreateOr(getShadow(&I, 0), getShadow(&I, 1));
2294 S = IRB.CreateBitCast(S, ResTy);
2295 S = IRB.CreateSExt(IRB.CreateICmpNE(S, Constant::getNullValue(ResTy)),
2296 ResTy);
2297 S = IRB.CreateBitCast(S, getShadowTy(&I));
2298 setShadow(&I, S);
2299 setOriginForNaryOp(I);
2300 }
2301
2302 // \brief Instrument compare-packed intrinsic.
2303 // Basically, an or followed by sext(icmp ne 0) to end up with all-zeros or
2304 // all-ones shadow.
handleVectorComparePackedIntrinsic__anon69a70b300211::MemorySanitizerVisitor2305 void handleVectorComparePackedIntrinsic(IntrinsicInst &I) {
2306 IRBuilder<> IRB(&I);
2307 Type *ResTy = getShadowTy(&I);
2308 Value *S0 = IRB.CreateOr(getShadow(&I, 0), getShadow(&I, 1));
2309 Value *S = IRB.CreateSExt(
2310 IRB.CreateICmpNE(S0, Constant::getNullValue(ResTy)), ResTy);
2311 setShadow(&I, S);
2312 setOriginForNaryOp(I);
2313 }
2314
2315 // \brief Instrument compare-scalar intrinsic.
2316 // This handles both cmp* intrinsics which return the result in the first
2317 // element of a vector, and comi* which return the result as i32.
handleVectorCompareScalarIntrinsic__anon69a70b300211::MemorySanitizerVisitor2318 void handleVectorCompareScalarIntrinsic(IntrinsicInst &I) {
2319 IRBuilder<> IRB(&I);
2320 Value *S0 = IRB.CreateOr(getShadow(&I, 0), getShadow(&I, 1));
2321 Value *S = LowerElementShadowExtend(IRB, S0, getShadowTy(&I));
2322 setShadow(&I, S);
2323 setOriginForNaryOp(I);
2324 }
2325
visitIntrinsicInst__anon69a70b300211::MemorySanitizerVisitor2326 void visitIntrinsicInst(IntrinsicInst &I) {
2327 switch (I.getIntrinsicID()) {
2328 case llvm::Intrinsic::bswap:
2329 handleBswap(I);
2330 break;
2331 case llvm::Intrinsic::x86_avx512_vcvtsd2usi64:
2332 case llvm::Intrinsic::x86_avx512_vcvtsd2usi32:
2333 case llvm::Intrinsic::x86_avx512_vcvtss2usi64:
2334 case llvm::Intrinsic::x86_avx512_vcvtss2usi32:
2335 case llvm::Intrinsic::x86_avx512_cvttss2usi64:
2336 case llvm::Intrinsic::x86_avx512_cvttss2usi:
2337 case llvm::Intrinsic::x86_avx512_cvttsd2usi64:
2338 case llvm::Intrinsic::x86_avx512_cvttsd2usi:
2339 case llvm::Intrinsic::x86_avx512_cvtusi2sd:
2340 case llvm::Intrinsic::x86_avx512_cvtusi2ss:
2341 case llvm::Intrinsic::x86_avx512_cvtusi642sd:
2342 case llvm::Intrinsic::x86_avx512_cvtusi642ss:
2343 case llvm::Intrinsic::x86_sse2_cvtsd2si64:
2344 case llvm::Intrinsic::x86_sse2_cvtsd2si:
2345 case llvm::Intrinsic::x86_sse2_cvtsd2ss:
2346 case llvm::Intrinsic::x86_sse2_cvtsi2sd:
2347 case llvm::Intrinsic::x86_sse2_cvtsi642sd:
2348 case llvm::Intrinsic::x86_sse2_cvtss2sd:
2349 case llvm::Intrinsic::x86_sse2_cvttsd2si64:
2350 case llvm::Intrinsic::x86_sse2_cvttsd2si:
2351 case llvm::Intrinsic::x86_sse_cvtsi2ss:
2352 case llvm::Intrinsic::x86_sse_cvtsi642ss:
2353 case llvm::Intrinsic::x86_sse_cvtss2si64:
2354 case llvm::Intrinsic::x86_sse_cvtss2si:
2355 case llvm::Intrinsic::x86_sse_cvttss2si64:
2356 case llvm::Intrinsic::x86_sse_cvttss2si:
2357 handleVectorConvertIntrinsic(I, 1);
2358 break;
2359 case llvm::Intrinsic::x86_sse_cvtps2pi:
2360 case llvm::Intrinsic::x86_sse_cvttps2pi:
2361 handleVectorConvertIntrinsic(I, 2);
2362 break;
2363 case llvm::Intrinsic::x86_avx2_psll_w:
2364 case llvm::Intrinsic::x86_avx2_psll_d:
2365 case llvm::Intrinsic::x86_avx2_psll_q:
2366 case llvm::Intrinsic::x86_avx2_pslli_w:
2367 case llvm::Intrinsic::x86_avx2_pslli_d:
2368 case llvm::Intrinsic::x86_avx2_pslli_q:
2369 case llvm::Intrinsic::x86_avx2_psrl_w:
2370 case llvm::Intrinsic::x86_avx2_psrl_d:
2371 case llvm::Intrinsic::x86_avx2_psrl_q:
2372 case llvm::Intrinsic::x86_avx2_psra_w:
2373 case llvm::Intrinsic::x86_avx2_psra_d:
2374 case llvm::Intrinsic::x86_avx2_psrli_w:
2375 case llvm::Intrinsic::x86_avx2_psrli_d:
2376 case llvm::Intrinsic::x86_avx2_psrli_q:
2377 case llvm::Intrinsic::x86_avx2_psrai_w:
2378 case llvm::Intrinsic::x86_avx2_psrai_d:
2379 case llvm::Intrinsic::x86_sse2_psll_w:
2380 case llvm::Intrinsic::x86_sse2_psll_d:
2381 case llvm::Intrinsic::x86_sse2_psll_q:
2382 case llvm::Intrinsic::x86_sse2_pslli_w:
2383 case llvm::Intrinsic::x86_sse2_pslli_d:
2384 case llvm::Intrinsic::x86_sse2_pslli_q:
2385 case llvm::Intrinsic::x86_sse2_psrl_w:
2386 case llvm::Intrinsic::x86_sse2_psrl_d:
2387 case llvm::Intrinsic::x86_sse2_psrl_q:
2388 case llvm::Intrinsic::x86_sse2_psra_w:
2389 case llvm::Intrinsic::x86_sse2_psra_d:
2390 case llvm::Intrinsic::x86_sse2_psrli_w:
2391 case llvm::Intrinsic::x86_sse2_psrli_d:
2392 case llvm::Intrinsic::x86_sse2_psrli_q:
2393 case llvm::Intrinsic::x86_sse2_psrai_w:
2394 case llvm::Intrinsic::x86_sse2_psrai_d:
2395 case llvm::Intrinsic::x86_mmx_psll_w:
2396 case llvm::Intrinsic::x86_mmx_psll_d:
2397 case llvm::Intrinsic::x86_mmx_psll_q:
2398 case llvm::Intrinsic::x86_mmx_pslli_w:
2399 case llvm::Intrinsic::x86_mmx_pslli_d:
2400 case llvm::Intrinsic::x86_mmx_pslli_q:
2401 case llvm::Intrinsic::x86_mmx_psrl_w:
2402 case llvm::Intrinsic::x86_mmx_psrl_d:
2403 case llvm::Intrinsic::x86_mmx_psrl_q:
2404 case llvm::Intrinsic::x86_mmx_psra_w:
2405 case llvm::Intrinsic::x86_mmx_psra_d:
2406 case llvm::Intrinsic::x86_mmx_psrli_w:
2407 case llvm::Intrinsic::x86_mmx_psrli_d:
2408 case llvm::Intrinsic::x86_mmx_psrli_q:
2409 case llvm::Intrinsic::x86_mmx_psrai_w:
2410 case llvm::Intrinsic::x86_mmx_psrai_d:
2411 handleVectorShiftIntrinsic(I, /* Variable */ false);
2412 break;
2413 case llvm::Intrinsic::x86_avx2_psllv_d:
2414 case llvm::Intrinsic::x86_avx2_psllv_d_256:
2415 case llvm::Intrinsic::x86_avx2_psllv_q:
2416 case llvm::Intrinsic::x86_avx2_psllv_q_256:
2417 case llvm::Intrinsic::x86_avx2_psrlv_d:
2418 case llvm::Intrinsic::x86_avx2_psrlv_d_256:
2419 case llvm::Intrinsic::x86_avx2_psrlv_q:
2420 case llvm::Intrinsic::x86_avx2_psrlv_q_256:
2421 case llvm::Intrinsic::x86_avx2_psrav_d:
2422 case llvm::Intrinsic::x86_avx2_psrav_d_256:
2423 handleVectorShiftIntrinsic(I, /* Variable */ true);
2424 break;
2425
2426 case llvm::Intrinsic::x86_sse2_packsswb_128:
2427 case llvm::Intrinsic::x86_sse2_packssdw_128:
2428 case llvm::Intrinsic::x86_sse2_packuswb_128:
2429 case llvm::Intrinsic::x86_sse41_packusdw:
2430 case llvm::Intrinsic::x86_avx2_packsswb:
2431 case llvm::Intrinsic::x86_avx2_packssdw:
2432 case llvm::Intrinsic::x86_avx2_packuswb:
2433 case llvm::Intrinsic::x86_avx2_packusdw:
2434 handleVectorPackIntrinsic(I);
2435 break;
2436
2437 case llvm::Intrinsic::x86_mmx_packsswb:
2438 case llvm::Intrinsic::x86_mmx_packuswb:
2439 handleVectorPackIntrinsic(I, 16);
2440 break;
2441
2442 case llvm::Intrinsic::x86_mmx_packssdw:
2443 handleVectorPackIntrinsic(I, 32);
2444 break;
2445
2446 case llvm::Intrinsic::x86_mmx_psad_bw:
2447 case llvm::Intrinsic::x86_sse2_psad_bw:
2448 case llvm::Intrinsic::x86_avx2_psad_bw:
2449 handleVectorSadIntrinsic(I);
2450 break;
2451
2452 case llvm::Intrinsic::x86_sse2_pmadd_wd:
2453 case llvm::Intrinsic::x86_avx2_pmadd_wd:
2454 case llvm::Intrinsic::x86_ssse3_pmadd_ub_sw_128:
2455 case llvm::Intrinsic::x86_avx2_pmadd_ub_sw:
2456 handleVectorPmaddIntrinsic(I);
2457 break;
2458
2459 case llvm::Intrinsic::x86_ssse3_pmadd_ub_sw:
2460 handleVectorPmaddIntrinsic(I, 8);
2461 break;
2462
2463 case llvm::Intrinsic::x86_mmx_pmadd_wd:
2464 handleVectorPmaddIntrinsic(I, 16);
2465 break;
2466
2467 case llvm::Intrinsic::x86_sse_cmp_ss:
2468 case llvm::Intrinsic::x86_sse2_cmp_sd:
2469 case llvm::Intrinsic::x86_sse_comieq_ss:
2470 case llvm::Intrinsic::x86_sse_comilt_ss:
2471 case llvm::Intrinsic::x86_sse_comile_ss:
2472 case llvm::Intrinsic::x86_sse_comigt_ss:
2473 case llvm::Intrinsic::x86_sse_comige_ss:
2474 case llvm::Intrinsic::x86_sse_comineq_ss:
2475 case llvm::Intrinsic::x86_sse_ucomieq_ss:
2476 case llvm::Intrinsic::x86_sse_ucomilt_ss:
2477 case llvm::Intrinsic::x86_sse_ucomile_ss:
2478 case llvm::Intrinsic::x86_sse_ucomigt_ss:
2479 case llvm::Intrinsic::x86_sse_ucomige_ss:
2480 case llvm::Intrinsic::x86_sse_ucomineq_ss:
2481 case llvm::Intrinsic::x86_sse2_comieq_sd:
2482 case llvm::Intrinsic::x86_sse2_comilt_sd:
2483 case llvm::Intrinsic::x86_sse2_comile_sd:
2484 case llvm::Intrinsic::x86_sse2_comigt_sd:
2485 case llvm::Intrinsic::x86_sse2_comige_sd:
2486 case llvm::Intrinsic::x86_sse2_comineq_sd:
2487 case llvm::Intrinsic::x86_sse2_ucomieq_sd:
2488 case llvm::Intrinsic::x86_sse2_ucomilt_sd:
2489 case llvm::Intrinsic::x86_sse2_ucomile_sd:
2490 case llvm::Intrinsic::x86_sse2_ucomigt_sd:
2491 case llvm::Intrinsic::x86_sse2_ucomige_sd:
2492 case llvm::Intrinsic::x86_sse2_ucomineq_sd:
2493 handleVectorCompareScalarIntrinsic(I);
2494 break;
2495
2496 case llvm::Intrinsic::x86_sse_cmp_ps:
2497 case llvm::Intrinsic::x86_sse2_cmp_pd:
2498 // FIXME: For x86_avx_cmp_pd_256 and x86_avx_cmp_ps_256 this function
2499 // generates reasonably looking IR that fails in the backend with "Do not
2500 // know how to split the result of this operator!".
2501 handleVectorComparePackedIntrinsic(I);
2502 break;
2503
2504 default:
2505 if (!handleUnknownIntrinsic(I))
2506 visitInstruction(I);
2507 break;
2508 }
2509 }
2510
visitCallSite__anon69a70b300211::MemorySanitizerVisitor2511 void visitCallSite(CallSite CS) {
2512 Instruction &I = *CS.getInstruction();
2513 assert((CS.isCall() || CS.isInvoke()) && "Unknown type of CallSite");
2514 if (CS.isCall()) {
2515 CallInst *Call = cast<CallInst>(&I);
2516
2517 // For inline asm, do the usual thing: check argument shadow and mark all
2518 // outputs as clean. Note that any side effects of the inline asm that are
2519 // not immediately visible in its constraints are not handled.
2520 if (Call->isInlineAsm()) {
2521 visitInstruction(I);
2522 return;
2523 }
2524
2525 assert(!isa<IntrinsicInst>(&I) && "intrinsics are handled elsewhere");
2526
2527 // We are going to insert code that relies on the fact that the callee
2528 // will become a non-readonly function after it is instrumented by us. To
2529 // prevent this code from being optimized out, mark that function
2530 // non-readonly in advance.
2531 if (Function *Func = Call->getCalledFunction()) {
2532 // Clear out readonly/readnone attributes.
2533 AttrBuilder B;
2534 B.addAttribute(Attribute::ReadOnly)
2535 .addAttribute(Attribute::ReadNone);
2536 Func->removeAttributes(AttributeSet::FunctionIndex,
2537 AttributeSet::get(Func->getContext(),
2538 AttributeSet::FunctionIndex,
2539 B));
2540 }
2541
2542 maybeMarkSanitizerLibraryCallNoBuiltin(Call, TLI);
2543 }
2544 IRBuilder<> IRB(&I);
2545
2546 unsigned ArgOffset = 0;
2547 DEBUG(dbgs() << " CallSite: " << I << "\n");
2548 for (CallSite::arg_iterator ArgIt = CS.arg_begin(), End = CS.arg_end();
2549 ArgIt != End; ++ArgIt) {
2550 Value *A = *ArgIt;
2551 unsigned i = ArgIt - CS.arg_begin();
2552 if (!A->getType()->isSized()) {
2553 DEBUG(dbgs() << "Arg " << i << " is not sized: " << I << "\n");
2554 continue;
2555 }
2556 unsigned Size = 0;
2557 Value *Store = nullptr;
2558 // Compute the Shadow for arg even if it is ByVal, because
2559 // in that case getShadow() will copy the actual arg shadow to
2560 // __msan_param_tls.
2561 Value *ArgShadow = getShadow(A);
2562 Value *ArgShadowBase = getShadowPtrForArgument(A, IRB, ArgOffset);
2563 DEBUG(dbgs() << " Arg#" << i << ": " << *A <<
2564 " Shadow: " << *ArgShadow << "\n");
2565 bool ArgIsInitialized = false;
2566 const DataLayout &DL = F.getParent()->getDataLayout();
2567 if (CS.paramHasAttr(i + 1, Attribute::ByVal)) {
2568 assert(A->getType()->isPointerTy() &&
2569 "ByVal argument is not a pointer!");
2570 Size = DL.getTypeAllocSize(A->getType()->getPointerElementType());
2571 if (ArgOffset + Size > kParamTLSSize) break;
2572 unsigned ParamAlignment = CS.getParamAlignment(i + 1);
2573 unsigned Alignment = std::min(ParamAlignment, kShadowTLSAlignment);
2574 Store = IRB.CreateMemCpy(ArgShadowBase,
2575 getShadowPtr(A, Type::getInt8Ty(*MS.C), IRB),
2576 Size, Alignment);
2577 } else {
2578 Size = DL.getTypeAllocSize(A->getType());
2579 if (ArgOffset + Size > kParamTLSSize) break;
2580 Store = IRB.CreateAlignedStore(ArgShadow, ArgShadowBase,
2581 kShadowTLSAlignment);
2582 Constant *Cst = dyn_cast<Constant>(ArgShadow);
2583 if (Cst && Cst->isNullValue()) ArgIsInitialized = true;
2584 }
2585 if (MS.TrackOrigins && !ArgIsInitialized)
2586 IRB.CreateStore(getOrigin(A),
2587 getOriginPtrForArgument(A, IRB, ArgOffset));
2588 (void)Store;
2589 assert(Size != 0 && Store != nullptr);
2590 DEBUG(dbgs() << " Param:" << *Store << "\n");
2591 ArgOffset += alignTo(Size, 8);
2592 }
2593 DEBUG(dbgs() << " done with call args\n");
2594
2595 FunctionType *FT =
2596 cast<FunctionType>(CS.getCalledValue()->getType()->getContainedType(0));
2597 if (FT->isVarArg()) {
2598 VAHelper->visitCallSite(CS, IRB);
2599 }
2600
2601 // Now, get the shadow for the RetVal.
2602 if (!I.getType()->isSized()) return;
2603 // Don't emit the epilogue for musttail call returns.
2604 if (CS.isCall() && cast<CallInst>(&I)->isMustTailCall()) return;
2605 IRBuilder<> IRBBefore(&I);
2606 // Until we have full dynamic coverage, make sure the retval shadow is 0.
2607 Value *Base = getShadowPtrForRetval(&I, IRBBefore);
2608 IRBBefore.CreateAlignedStore(getCleanShadow(&I), Base, kShadowTLSAlignment);
2609 BasicBlock::iterator NextInsn;
2610 if (CS.isCall()) {
2611 NextInsn = ++I.getIterator();
2612 assert(NextInsn != I.getParent()->end());
2613 } else {
2614 BasicBlock *NormalDest = cast<InvokeInst>(&I)->getNormalDest();
2615 if (!NormalDest->getSinglePredecessor()) {
2616 // FIXME: this case is tricky, so we are just conservative here.
2617 // Perhaps we need to split the edge between this BB and NormalDest,
2618 // but a naive attempt to use SplitEdge leads to a crash.
2619 setShadow(&I, getCleanShadow(&I));
2620 setOrigin(&I, getCleanOrigin());
2621 return;
2622 }
2623 NextInsn = NormalDest->getFirstInsertionPt();
2624 assert(NextInsn != NormalDest->end() &&
2625 "Could not find insertion point for retval shadow load");
2626 }
2627 IRBuilder<> IRBAfter(&*NextInsn);
2628 Value *RetvalShadow =
2629 IRBAfter.CreateAlignedLoad(getShadowPtrForRetval(&I, IRBAfter),
2630 kShadowTLSAlignment, "_msret");
2631 setShadow(&I, RetvalShadow);
2632 if (MS.TrackOrigins)
2633 setOrigin(&I, IRBAfter.CreateLoad(getOriginPtrForRetval(IRBAfter)));
2634 }
2635
isAMustTailRetVal__anon69a70b300211::MemorySanitizerVisitor2636 bool isAMustTailRetVal(Value *RetVal) {
2637 if (auto *I = dyn_cast<BitCastInst>(RetVal)) {
2638 RetVal = I->getOperand(0);
2639 }
2640 if (auto *I = dyn_cast<CallInst>(RetVal)) {
2641 return I->isMustTailCall();
2642 }
2643 return false;
2644 }
2645
visitReturnInst__anon69a70b300211::MemorySanitizerVisitor2646 void visitReturnInst(ReturnInst &I) {
2647 IRBuilder<> IRB(&I);
2648 Value *RetVal = I.getReturnValue();
2649 if (!RetVal) return;
2650 // Don't emit the epilogue for musttail call returns.
2651 if (isAMustTailRetVal(RetVal)) return;
2652 Value *ShadowPtr = getShadowPtrForRetval(RetVal, IRB);
2653 if (CheckReturnValue) {
2654 insertShadowCheck(RetVal, &I);
2655 Value *Shadow = getCleanShadow(RetVal);
2656 IRB.CreateAlignedStore(Shadow, ShadowPtr, kShadowTLSAlignment);
2657 } else {
2658 Value *Shadow = getShadow(RetVal);
2659 IRB.CreateAlignedStore(Shadow, ShadowPtr, kShadowTLSAlignment);
2660 // FIXME: make it conditional if ClStoreCleanOrigin==0
2661 if (MS.TrackOrigins)
2662 IRB.CreateStore(getOrigin(RetVal), getOriginPtrForRetval(IRB));
2663 }
2664 }
2665
visitPHINode__anon69a70b300211::MemorySanitizerVisitor2666 void visitPHINode(PHINode &I) {
2667 IRBuilder<> IRB(&I);
2668 if (!PropagateShadow) {
2669 setShadow(&I, getCleanShadow(&I));
2670 setOrigin(&I, getCleanOrigin());
2671 return;
2672 }
2673
2674 ShadowPHINodes.push_back(&I);
2675 setShadow(&I, IRB.CreatePHI(getShadowTy(&I), I.getNumIncomingValues(),
2676 "_msphi_s"));
2677 if (MS.TrackOrigins)
2678 setOrigin(&I, IRB.CreatePHI(MS.OriginTy, I.getNumIncomingValues(),
2679 "_msphi_o"));
2680 }
2681
visitAllocaInst__anon69a70b300211::MemorySanitizerVisitor2682 void visitAllocaInst(AllocaInst &I) {
2683 setShadow(&I, getCleanShadow(&I));
2684 setOrigin(&I, getCleanOrigin());
2685 IRBuilder<> IRB(I.getNextNode());
2686 const DataLayout &DL = F.getParent()->getDataLayout();
2687 uint64_t Size = DL.getTypeAllocSize(I.getAllocatedType());
2688 if (PoisonStack && ClPoisonStackWithCall) {
2689 IRB.CreateCall(MS.MsanPoisonStackFn,
2690 {IRB.CreatePointerCast(&I, IRB.getInt8PtrTy()),
2691 ConstantInt::get(MS.IntptrTy, Size)});
2692 } else {
2693 Value *ShadowBase = getShadowPtr(&I, Type::getInt8PtrTy(*MS.C), IRB);
2694 Value *PoisonValue = IRB.getInt8(PoisonStack ? ClPoisonStackPattern : 0);
2695 IRB.CreateMemSet(ShadowBase, PoisonValue, Size, I.getAlignment());
2696 }
2697
2698 if (PoisonStack && MS.TrackOrigins) {
2699 SmallString<2048> StackDescriptionStorage;
2700 raw_svector_ostream StackDescription(StackDescriptionStorage);
2701 // We create a string with a description of the stack allocation and
2702 // pass it into __msan_set_alloca_origin.
2703 // It will be printed by the run-time if stack-originated UMR is found.
2704 // The first 4 bytes of the string are set to '----' and will be replaced
2705 // by __msan_va_arg_overflow_size_tls at the first call.
2706 StackDescription << "----" << I.getName() << "@" << F.getName();
2707 Value *Descr =
2708 createPrivateNonConstGlobalForString(*F.getParent(),
2709 StackDescription.str());
2710
2711 IRB.CreateCall(MS.MsanSetAllocaOrigin4Fn,
2712 {IRB.CreatePointerCast(&I, IRB.getInt8PtrTy()),
2713 ConstantInt::get(MS.IntptrTy, Size),
2714 IRB.CreatePointerCast(Descr, IRB.getInt8PtrTy()),
2715 IRB.CreatePointerCast(&F, MS.IntptrTy)});
2716 }
2717 }
2718
visitSelectInst__anon69a70b300211::MemorySanitizerVisitor2719 void visitSelectInst(SelectInst& I) {
2720 IRBuilder<> IRB(&I);
2721 // a = select b, c, d
2722 Value *B = I.getCondition();
2723 Value *C = I.getTrueValue();
2724 Value *D = I.getFalseValue();
2725 Value *Sb = getShadow(B);
2726 Value *Sc = getShadow(C);
2727 Value *Sd = getShadow(D);
2728
2729 // Result shadow if condition shadow is 0.
2730 Value *Sa0 = IRB.CreateSelect(B, Sc, Sd);
2731 Value *Sa1;
2732 if (I.getType()->isAggregateType()) {
2733 // To avoid "sign extending" i1 to an arbitrary aggregate type, we just do
2734 // an extra "select". This results in much more compact IR.
2735 // Sa = select Sb, poisoned, (select b, Sc, Sd)
2736 Sa1 = getPoisonedShadow(getShadowTy(I.getType()));
2737 } else {
2738 // Sa = select Sb, [ (c^d) | Sc | Sd ], [ b ? Sc : Sd ]
2739 // If Sb (condition is poisoned), look for bits in c and d that are equal
2740 // and both unpoisoned.
2741 // If !Sb (condition is unpoisoned), simply pick one of Sc and Sd.
2742
2743 // Cast arguments to shadow-compatible type.
2744 C = CreateAppToShadowCast(IRB, C);
2745 D = CreateAppToShadowCast(IRB, D);
2746
2747 // Result shadow if condition shadow is 1.
2748 Sa1 = IRB.CreateOr(IRB.CreateXor(C, D), IRB.CreateOr(Sc, Sd));
2749 }
2750 Value *Sa = IRB.CreateSelect(Sb, Sa1, Sa0, "_msprop_select");
2751 setShadow(&I, Sa);
2752 if (MS.TrackOrigins) {
2753 // Origins are always i32, so any vector conditions must be flattened.
2754 // FIXME: consider tracking vector origins for app vectors?
2755 if (B->getType()->isVectorTy()) {
2756 Type *FlatTy = getShadowTyNoVec(B->getType());
2757 B = IRB.CreateICmpNE(IRB.CreateBitCast(B, FlatTy),
2758 ConstantInt::getNullValue(FlatTy));
2759 Sb = IRB.CreateICmpNE(IRB.CreateBitCast(Sb, FlatTy),
2760 ConstantInt::getNullValue(FlatTy));
2761 }
2762 // a = select b, c, d
2763 // Oa = Sb ? Ob : (b ? Oc : Od)
2764 setOrigin(
2765 &I, IRB.CreateSelect(Sb, getOrigin(I.getCondition()),
2766 IRB.CreateSelect(B, getOrigin(I.getTrueValue()),
2767 getOrigin(I.getFalseValue()))));
2768 }
2769 }
2770
visitLandingPadInst__anon69a70b300211::MemorySanitizerVisitor2771 void visitLandingPadInst(LandingPadInst &I) {
2772 // Do nothing.
2773 // See http://code.google.com/p/memory-sanitizer/issues/detail?id=1
2774 setShadow(&I, getCleanShadow(&I));
2775 setOrigin(&I, getCleanOrigin());
2776 }
2777
visitCatchSwitchInst__anon69a70b300211::MemorySanitizerVisitor2778 void visitCatchSwitchInst(CatchSwitchInst &I) {
2779 setShadow(&I, getCleanShadow(&I));
2780 setOrigin(&I, getCleanOrigin());
2781 }
2782
visitFuncletPadInst__anon69a70b300211::MemorySanitizerVisitor2783 void visitFuncletPadInst(FuncletPadInst &I) {
2784 setShadow(&I, getCleanShadow(&I));
2785 setOrigin(&I, getCleanOrigin());
2786 }
2787
visitGetElementPtrInst__anon69a70b300211::MemorySanitizerVisitor2788 void visitGetElementPtrInst(GetElementPtrInst &I) {
2789 handleShadowOr(I);
2790 }
2791
visitExtractValueInst__anon69a70b300211::MemorySanitizerVisitor2792 void visitExtractValueInst(ExtractValueInst &I) {
2793 IRBuilder<> IRB(&I);
2794 Value *Agg = I.getAggregateOperand();
2795 DEBUG(dbgs() << "ExtractValue: " << I << "\n");
2796 Value *AggShadow = getShadow(Agg);
2797 DEBUG(dbgs() << " AggShadow: " << *AggShadow << "\n");
2798 Value *ResShadow = IRB.CreateExtractValue(AggShadow, I.getIndices());
2799 DEBUG(dbgs() << " ResShadow: " << *ResShadow << "\n");
2800 setShadow(&I, ResShadow);
2801 setOriginForNaryOp(I);
2802 }
2803
visitInsertValueInst__anon69a70b300211::MemorySanitizerVisitor2804 void visitInsertValueInst(InsertValueInst &I) {
2805 IRBuilder<> IRB(&I);
2806 DEBUG(dbgs() << "InsertValue: " << I << "\n");
2807 Value *AggShadow = getShadow(I.getAggregateOperand());
2808 Value *InsShadow = getShadow(I.getInsertedValueOperand());
2809 DEBUG(dbgs() << " AggShadow: " << *AggShadow << "\n");
2810 DEBUG(dbgs() << " InsShadow: " << *InsShadow << "\n");
2811 Value *Res = IRB.CreateInsertValue(AggShadow, InsShadow, I.getIndices());
2812 DEBUG(dbgs() << " Res: " << *Res << "\n");
2813 setShadow(&I, Res);
2814 setOriginForNaryOp(I);
2815 }
2816
dumpInst__anon69a70b300211::MemorySanitizerVisitor2817 void dumpInst(Instruction &I) {
2818 if (CallInst *CI = dyn_cast<CallInst>(&I)) {
2819 errs() << "ZZZ call " << CI->getCalledFunction()->getName() << "\n";
2820 } else {
2821 errs() << "ZZZ " << I.getOpcodeName() << "\n";
2822 }
2823 errs() << "QQQ " << I << "\n";
2824 }
2825
visitResumeInst__anon69a70b300211::MemorySanitizerVisitor2826 void visitResumeInst(ResumeInst &I) {
2827 DEBUG(dbgs() << "Resume: " << I << "\n");
2828 // Nothing to do here.
2829 }
2830
visitCleanupReturnInst__anon69a70b300211::MemorySanitizerVisitor2831 void visitCleanupReturnInst(CleanupReturnInst &CRI) {
2832 DEBUG(dbgs() << "CleanupReturn: " << CRI << "\n");
2833 // Nothing to do here.
2834 }
2835
visitCatchReturnInst__anon69a70b300211::MemorySanitizerVisitor2836 void visitCatchReturnInst(CatchReturnInst &CRI) {
2837 DEBUG(dbgs() << "CatchReturn: " << CRI << "\n");
2838 // Nothing to do here.
2839 }
2840
visitInstruction__anon69a70b300211::MemorySanitizerVisitor2841 void visitInstruction(Instruction &I) {
2842 // Everything else: stop propagating and check for poisoned shadow.
2843 if (ClDumpStrictInstructions)
2844 dumpInst(I);
2845 DEBUG(dbgs() << "DEFAULT: " << I << "\n");
2846 for (size_t i = 0, n = I.getNumOperands(); i < n; i++)
2847 insertShadowCheck(I.getOperand(i), &I);
2848 setShadow(&I, getCleanShadow(&I));
2849 setOrigin(&I, getCleanOrigin());
2850 }
2851 };
2852
2853 /// \brief AMD64-specific implementation of VarArgHelper.
2854 struct VarArgAMD64Helper : public VarArgHelper {
2855 // An unfortunate workaround for asymmetric lowering of va_arg stuff.
2856 // See a comment in visitCallSite for more details.
2857 static const unsigned AMD64GpEndOffset = 48; // AMD64 ABI Draft 0.99.6 p3.5.7
2858 static const unsigned AMD64FpEndOffset = 176;
2859
2860 Function &F;
2861 MemorySanitizer &MS;
2862 MemorySanitizerVisitor &MSV;
2863 Value *VAArgTLSCopy;
2864 Value *VAArgOverflowSize;
2865
2866 SmallVector<CallInst*, 16> VAStartInstrumentationList;
2867
VarArgAMD64Helper__anon69a70b300211::VarArgAMD64Helper2868 VarArgAMD64Helper(Function &F, MemorySanitizer &MS,
2869 MemorySanitizerVisitor &MSV)
2870 : F(F), MS(MS), MSV(MSV), VAArgTLSCopy(nullptr),
2871 VAArgOverflowSize(nullptr) {}
2872
2873 enum ArgKind { AK_GeneralPurpose, AK_FloatingPoint, AK_Memory };
2874
classifyArgument__anon69a70b300211::VarArgAMD64Helper2875 ArgKind classifyArgument(Value* arg) {
2876 // A very rough approximation of X86_64 argument classification rules.
2877 Type *T = arg->getType();
2878 if (T->isFPOrFPVectorTy() || T->isX86_MMXTy())
2879 return AK_FloatingPoint;
2880 if (T->isIntegerTy() && T->getPrimitiveSizeInBits() <= 64)
2881 return AK_GeneralPurpose;
2882 if (T->isPointerTy())
2883 return AK_GeneralPurpose;
2884 return AK_Memory;
2885 }
2886
2887 // For VarArg functions, store the argument shadow in an ABI-specific format
2888 // that corresponds to va_list layout.
2889 // We do this because Clang lowers va_arg in the frontend, and this pass
2890 // only sees the low level code that deals with va_list internals.
2891 // A much easier alternative (provided that Clang emits va_arg instructions)
2892 // would have been to associate each live instance of va_list with a copy of
2893 // MSanParamTLS, and extract shadow on va_arg() call in the argument list
2894 // order.
visitCallSite__anon69a70b300211::VarArgAMD64Helper2895 void visitCallSite(CallSite &CS, IRBuilder<> &IRB) override {
2896 unsigned GpOffset = 0;
2897 unsigned FpOffset = AMD64GpEndOffset;
2898 unsigned OverflowOffset = AMD64FpEndOffset;
2899 const DataLayout &DL = F.getParent()->getDataLayout();
2900 for (CallSite::arg_iterator ArgIt = CS.arg_begin(), End = CS.arg_end();
2901 ArgIt != End; ++ArgIt) {
2902 Value *A = *ArgIt;
2903 unsigned ArgNo = CS.getArgumentNo(ArgIt);
2904 bool IsFixed = ArgNo < CS.getFunctionType()->getNumParams();
2905 bool IsByVal = CS.paramHasAttr(ArgNo + 1, Attribute::ByVal);
2906 if (IsByVal) {
2907 // ByVal arguments always go to the overflow area.
2908 // Fixed arguments passed through the overflow area will be stepped
2909 // over by va_start, so don't count them towards the offset.
2910 if (IsFixed)
2911 continue;
2912 assert(A->getType()->isPointerTy());
2913 Type *RealTy = A->getType()->getPointerElementType();
2914 uint64_t ArgSize = DL.getTypeAllocSize(RealTy);
2915 Value *Base = getShadowPtrForVAArgument(RealTy, IRB, OverflowOffset);
2916 OverflowOffset += alignTo(ArgSize, 8);
2917 IRB.CreateMemCpy(Base, MSV.getShadowPtr(A, IRB.getInt8Ty(), IRB),
2918 ArgSize, kShadowTLSAlignment);
2919 } else {
2920 ArgKind AK = classifyArgument(A);
2921 if (AK == AK_GeneralPurpose && GpOffset >= AMD64GpEndOffset)
2922 AK = AK_Memory;
2923 if (AK == AK_FloatingPoint && FpOffset >= AMD64FpEndOffset)
2924 AK = AK_Memory;
2925 Value *Base;
2926 switch (AK) {
2927 case AK_GeneralPurpose:
2928 Base = getShadowPtrForVAArgument(A->getType(), IRB, GpOffset);
2929 GpOffset += 8;
2930 break;
2931 case AK_FloatingPoint:
2932 Base = getShadowPtrForVAArgument(A->getType(), IRB, FpOffset);
2933 FpOffset += 16;
2934 break;
2935 case AK_Memory:
2936 if (IsFixed)
2937 continue;
2938 uint64_t ArgSize = DL.getTypeAllocSize(A->getType());
2939 Base = getShadowPtrForVAArgument(A->getType(), IRB, OverflowOffset);
2940 OverflowOffset += alignTo(ArgSize, 8);
2941 }
2942 // Take fixed arguments into account for GpOffset and FpOffset,
2943 // but don't actually store shadows for them.
2944 if (IsFixed)
2945 continue;
2946 IRB.CreateAlignedStore(MSV.getShadow(A), Base, kShadowTLSAlignment);
2947 }
2948 }
2949 Constant *OverflowSize =
2950 ConstantInt::get(IRB.getInt64Ty(), OverflowOffset - AMD64FpEndOffset);
2951 IRB.CreateStore(OverflowSize, MS.VAArgOverflowSizeTLS);
2952 }
2953
2954 /// \brief Compute the shadow address for a given va_arg.
getShadowPtrForVAArgument__anon69a70b300211::VarArgAMD64Helper2955 Value *getShadowPtrForVAArgument(Type *Ty, IRBuilder<> &IRB,
2956 int ArgOffset) {
2957 Value *Base = IRB.CreatePointerCast(MS.VAArgTLS, MS.IntptrTy);
2958 Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
2959 return IRB.CreateIntToPtr(Base, PointerType::get(MSV.getShadowTy(Ty), 0),
2960 "_msarg");
2961 }
2962
visitVAStartInst__anon69a70b300211::VarArgAMD64Helper2963 void visitVAStartInst(VAStartInst &I) override {
2964 if (F.getCallingConv() == CallingConv::X86_64_Win64)
2965 return;
2966 IRBuilder<> IRB(&I);
2967 VAStartInstrumentationList.push_back(&I);
2968 Value *VAListTag = I.getArgOperand(0);
2969 Value *ShadowPtr = MSV.getShadowPtr(VAListTag, IRB.getInt8Ty(), IRB);
2970
2971 // Unpoison the whole __va_list_tag.
2972 // FIXME: magic ABI constants.
2973 IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
2974 /* size */24, /* alignment */8, false);
2975 }
2976
visitVACopyInst__anon69a70b300211::VarArgAMD64Helper2977 void visitVACopyInst(VACopyInst &I) override {
2978 if (F.getCallingConv() == CallingConv::X86_64_Win64)
2979 return;
2980 IRBuilder<> IRB(&I);
2981 Value *VAListTag = I.getArgOperand(0);
2982 Value *ShadowPtr = MSV.getShadowPtr(VAListTag, IRB.getInt8Ty(), IRB);
2983
2984 // Unpoison the whole __va_list_tag.
2985 // FIXME: magic ABI constants.
2986 IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
2987 /* size */24, /* alignment */8, false);
2988 }
2989
finalizeInstrumentation__anon69a70b300211::VarArgAMD64Helper2990 void finalizeInstrumentation() override {
2991 assert(!VAArgOverflowSize && !VAArgTLSCopy &&
2992 "finalizeInstrumentation called twice");
2993 if (!VAStartInstrumentationList.empty()) {
2994 // If there is a va_start in this function, make a backup copy of
2995 // va_arg_tls somewhere in the function entry block.
2996 IRBuilder<> IRB(F.getEntryBlock().getFirstNonPHI());
2997 VAArgOverflowSize = IRB.CreateLoad(MS.VAArgOverflowSizeTLS);
2998 Value *CopySize =
2999 IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, AMD64FpEndOffset),
3000 VAArgOverflowSize);
3001 VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
3002 IRB.CreateMemCpy(VAArgTLSCopy, MS.VAArgTLS, CopySize, 8);
3003 }
3004
3005 // Instrument va_start.
3006 // Copy va_list shadow from the backup copy of the TLS contents.
3007 for (size_t i = 0, n = VAStartInstrumentationList.size(); i < n; i++) {
3008 CallInst *OrigInst = VAStartInstrumentationList[i];
3009 IRBuilder<> IRB(OrigInst->getNextNode());
3010 Value *VAListTag = OrigInst->getArgOperand(0);
3011
3012 Value *RegSaveAreaPtrPtr =
3013 IRB.CreateIntToPtr(
3014 IRB.CreateAdd(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
3015 ConstantInt::get(MS.IntptrTy, 16)),
3016 Type::getInt64PtrTy(*MS.C));
3017 Value *RegSaveAreaPtr = IRB.CreateLoad(RegSaveAreaPtrPtr);
3018 Value *RegSaveAreaShadowPtr =
3019 MSV.getShadowPtr(RegSaveAreaPtr, IRB.getInt8Ty(), IRB);
3020 IRB.CreateMemCpy(RegSaveAreaShadowPtr, VAArgTLSCopy,
3021 AMD64FpEndOffset, 16);
3022
3023 Value *OverflowArgAreaPtrPtr =
3024 IRB.CreateIntToPtr(
3025 IRB.CreateAdd(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
3026 ConstantInt::get(MS.IntptrTy, 8)),
3027 Type::getInt64PtrTy(*MS.C));
3028 Value *OverflowArgAreaPtr = IRB.CreateLoad(OverflowArgAreaPtrPtr);
3029 Value *OverflowArgAreaShadowPtr =
3030 MSV.getShadowPtr(OverflowArgAreaPtr, IRB.getInt8Ty(), IRB);
3031 Value *SrcPtr = IRB.CreateConstGEP1_32(IRB.getInt8Ty(), VAArgTLSCopy,
3032 AMD64FpEndOffset);
3033 IRB.CreateMemCpy(OverflowArgAreaShadowPtr, SrcPtr, VAArgOverflowSize, 16);
3034 }
3035 }
3036 };
3037
3038 /// \brief MIPS64-specific implementation of VarArgHelper.
3039 struct VarArgMIPS64Helper : public VarArgHelper {
3040 Function &F;
3041 MemorySanitizer &MS;
3042 MemorySanitizerVisitor &MSV;
3043 Value *VAArgTLSCopy;
3044 Value *VAArgSize;
3045
3046 SmallVector<CallInst*, 16> VAStartInstrumentationList;
3047
VarArgMIPS64Helper__anon69a70b300211::VarArgMIPS64Helper3048 VarArgMIPS64Helper(Function &F, MemorySanitizer &MS,
3049 MemorySanitizerVisitor &MSV)
3050 : F(F), MS(MS), MSV(MSV), VAArgTLSCopy(nullptr),
3051 VAArgSize(nullptr) {}
3052
visitCallSite__anon69a70b300211::VarArgMIPS64Helper3053 void visitCallSite(CallSite &CS, IRBuilder<> &IRB) override {
3054 unsigned VAArgOffset = 0;
3055 const DataLayout &DL = F.getParent()->getDataLayout();
3056 for (CallSite::arg_iterator ArgIt = CS.arg_begin() +
3057 CS.getFunctionType()->getNumParams(), End = CS.arg_end();
3058 ArgIt != End; ++ArgIt) {
3059 llvm::Triple TargetTriple(F.getParent()->getTargetTriple());
3060 Value *A = *ArgIt;
3061 Value *Base;
3062 uint64_t ArgSize = DL.getTypeAllocSize(A->getType());
3063 if (TargetTriple.getArch() == llvm::Triple::mips64) {
3064 // Adjusting the shadow for argument with size < 8 to match the placement
3065 // of bits in big endian system
3066 if (ArgSize < 8)
3067 VAArgOffset += (8 - ArgSize);
3068 }
3069 Base = getShadowPtrForVAArgument(A->getType(), IRB, VAArgOffset);
3070 VAArgOffset += ArgSize;
3071 VAArgOffset = alignTo(VAArgOffset, 8);
3072 IRB.CreateAlignedStore(MSV.getShadow(A), Base, kShadowTLSAlignment);
3073 }
3074
3075 Constant *TotalVAArgSize = ConstantInt::get(IRB.getInt64Ty(), VAArgOffset);
3076 // Here using VAArgOverflowSizeTLS as VAArgSizeTLS to avoid creation of
3077 // a new class member i.e. it is the total size of all VarArgs.
3078 IRB.CreateStore(TotalVAArgSize, MS.VAArgOverflowSizeTLS);
3079 }
3080
3081 /// \brief Compute the shadow address for a given va_arg.
getShadowPtrForVAArgument__anon69a70b300211::VarArgMIPS64Helper3082 Value *getShadowPtrForVAArgument(Type *Ty, IRBuilder<> &IRB,
3083 int ArgOffset) {
3084 Value *Base = IRB.CreatePointerCast(MS.VAArgTLS, MS.IntptrTy);
3085 Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
3086 return IRB.CreateIntToPtr(Base, PointerType::get(MSV.getShadowTy(Ty), 0),
3087 "_msarg");
3088 }
3089
visitVAStartInst__anon69a70b300211::VarArgMIPS64Helper3090 void visitVAStartInst(VAStartInst &I) override {
3091 IRBuilder<> IRB(&I);
3092 VAStartInstrumentationList.push_back(&I);
3093 Value *VAListTag = I.getArgOperand(0);
3094 Value *ShadowPtr = MSV.getShadowPtr(VAListTag, IRB.getInt8Ty(), IRB);
3095 IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
3096 /* size */8, /* alignment */8, false);
3097 }
3098
visitVACopyInst__anon69a70b300211::VarArgMIPS64Helper3099 void visitVACopyInst(VACopyInst &I) override {
3100 IRBuilder<> IRB(&I);
3101 Value *VAListTag = I.getArgOperand(0);
3102 Value *ShadowPtr = MSV.getShadowPtr(VAListTag, IRB.getInt8Ty(), IRB);
3103 // Unpoison the whole __va_list_tag.
3104 // FIXME: magic ABI constants.
3105 IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
3106 /* size */8, /* alignment */8, false);
3107 }
3108
finalizeInstrumentation__anon69a70b300211::VarArgMIPS64Helper3109 void finalizeInstrumentation() override {
3110 assert(!VAArgSize && !VAArgTLSCopy &&
3111 "finalizeInstrumentation called twice");
3112 IRBuilder<> IRB(F.getEntryBlock().getFirstNonPHI());
3113 VAArgSize = IRB.CreateLoad(MS.VAArgOverflowSizeTLS);
3114 Value *CopySize = IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, 0),
3115 VAArgSize);
3116
3117 if (!VAStartInstrumentationList.empty()) {
3118 // If there is a va_start in this function, make a backup copy of
3119 // va_arg_tls somewhere in the function entry block.
3120 VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
3121 IRB.CreateMemCpy(VAArgTLSCopy, MS.VAArgTLS, CopySize, 8);
3122 }
3123
3124 // Instrument va_start.
3125 // Copy va_list shadow from the backup copy of the TLS contents.
3126 for (size_t i = 0, n = VAStartInstrumentationList.size(); i < n; i++) {
3127 CallInst *OrigInst = VAStartInstrumentationList[i];
3128 IRBuilder<> IRB(OrigInst->getNextNode());
3129 Value *VAListTag = OrigInst->getArgOperand(0);
3130 Value *RegSaveAreaPtrPtr =
3131 IRB.CreateIntToPtr(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
3132 Type::getInt64PtrTy(*MS.C));
3133 Value *RegSaveAreaPtr = IRB.CreateLoad(RegSaveAreaPtrPtr);
3134 Value *RegSaveAreaShadowPtr =
3135 MSV.getShadowPtr(RegSaveAreaPtr, IRB.getInt8Ty(), IRB);
3136 IRB.CreateMemCpy(RegSaveAreaShadowPtr, VAArgTLSCopy, CopySize, 8);
3137 }
3138 }
3139 };
3140
3141
3142 /// \brief AArch64-specific implementation of VarArgHelper.
3143 struct VarArgAArch64Helper : public VarArgHelper {
3144 static const unsigned kAArch64GrArgSize = 64;
3145 static const unsigned kAArch64VrArgSize = 128;
3146
3147 static const unsigned AArch64GrBegOffset = 0;
3148 static const unsigned AArch64GrEndOffset = kAArch64GrArgSize;
3149 // Make VR space aligned to 16 bytes.
3150 static const unsigned AArch64VrBegOffset = AArch64GrEndOffset;
3151 static const unsigned AArch64VrEndOffset = AArch64VrBegOffset
3152 + kAArch64VrArgSize;
3153 static const unsigned AArch64VAEndOffset = AArch64VrEndOffset;
3154
3155 Function &F;
3156 MemorySanitizer &MS;
3157 MemorySanitizerVisitor &MSV;
3158 Value *VAArgTLSCopy;
3159 Value *VAArgOverflowSize;
3160
3161 SmallVector<CallInst*, 16> VAStartInstrumentationList;
3162
VarArgAArch64Helper__anon69a70b300211::VarArgAArch64Helper3163 VarArgAArch64Helper(Function &F, MemorySanitizer &MS,
3164 MemorySanitizerVisitor &MSV)
3165 : F(F), MS(MS), MSV(MSV), VAArgTLSCopy(nullptr),
3166 VAArgOverflowSize(nullptr) {}
3167
3168 enum ArgKind { AK_GeneralPurpose, AK_FloatingPoint, AK_Memory };
3169
classifyArgument__anon69a70b300211::VarArgAArch64Helper3170 ArgKind classifyArgument(Value* arg) {
3171 Type *T = arg->getType();
3172 if (T->isFPOrFPVectorTy())
3173 return AK_FloatingPoint;
3174 if ((T->isIntegerTy() && T->getPrimitiveSizeInBits() <= 64)
3175 || (T->isPointerTy()))
3176 return AK_GeneralPurpose;
3177 return AK_Memory;
3178 }
3179
3180 // The instrumentation stores the argument shadow in a non ABI-specific
3181 // format because it does not know which argument is named (since Clang,
3182 // like x86_64 case, lowers the va_args in the frontend and this pass only
3183 // sees the low level code that deals with va_list internals).
3184 // The first seven GR registers are saved in the first 56 bytes of the
3185 // va_arg tls arra, followers by the first 8 FP/SIMD registers, and then
3186 // the remaining arguments.
3187 // Using constant offset within the va_arg TLS array allows fast copy
3188 // in the finalize instrumentation.
visitCallSite__anon69a70b300211::VarArgAArch64Helper3189 void visitCallSite(CallSite &CS, IRBuilder<> &IRB) override {
3190 unsigned GrOffset = AArch64GrBegOffset;
3191 unsigned VrOffset = AArch64VrBegOffset;
3192 unsigned OverflowOffset = AArch64VAEndOffset;
3193
3194 const DataLayout &DL = F.getParent()->getDataLayout();
3195 for (CallSite::arg_iterator ArgIt = CS.arg_begin(), End = CS.arg_end();
3196 ArgIt != End; ++ArgIt) {
3197 Value *A = *ArgIt;
3198 unsigned ArgNo = CS.getArgumentNo(ArgIt);
3199 bool IsFixed = ArgNo < CS.getFunctionType()->getNumParams();
3200 ArgKind AK = classifyArgument(A);
3201 if (AK == AK_GeneralPurpose && GrOffset >= AArch64GrEndOffset)
3202 AK = AK_Memory;
3203 if (AK == AK_FloatingPoint && VrOffset >= AArch64VrEndOffset)
3204 AK = AK_Memory;
3205 Value *Base;
3206 switch (AK) {
3207 case AK_GeneralPurpose:
3208 Base = getShadowPtrForVAArgument(A->getType(), IRB, GrOffset);
3209 GrOffset += 8;
3210 break;
3211 case AK_FloatingPoint:
3212 Base = getShadowPtrForVAArgument(A->getType(), IRB, VrOffset);
3213 VrOffset += 16;
3214 break;
3215 case AK_Memory:
3216 // Don't count fixed arguments in the overflow area - va_start will
3217 // skip right over them.
3218 if (IsFixed)
3219 continue;
3220 uint64_t ArgSize = DL.getTypeAllocSize(A->getType());
3221 Base = getShadowPtrForVAArgument(A->getType(), IRB, OverflowOffset);
3222 OverflowOffset += alignTo(ArgSize, 8);
3223 break;
3224 }
3225 // Count Gp/Vr fixed arguments to their respective offsets, but don't
3226 // bother to actually store a shadow.
3227 if (IsFixed)
3228 continue;
3229 IRB.CreateAlignedStore(MSV.getShadow(A), Base, kShadowTLSAlignment);
3230 }
3231 Constant *OverflowSize =
3232 ConstantInt::get(IRB.getInt64Ty(), OverflowOffset - AArch64VAEndOffset);
3233 IRB.CreateStore(OverflowSize, MS.VAArgOverflowSizeTLS);
3234 }
3235
3236 /// Compute the shadow address for a given va_arg.
getShadowPtrForVAArgument__anon69a70b300211::VarArgAArch64Helper3237 Value *getShadowPtrForVAArgument(Type *Ty, IRBuilder<> &IRB,
3238 int ArgOffset) {
3239 Value *Base = IRB.CreatePointerCast(MS.VAArgTLS, MS.IntptrTy);
3240 Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
3241 return IRB.CreateIntToPtr(Base, PointerType::get(MSV.getShadowTy(Ty), 0),
3242 "_msarg");
3243 }
3244
visitVAStartInst__anon69a70b300211::VarArgAArch64Helper3245 void visitVAStartInst(VAStartInst &I) override {
3246 IRBuilder<> IRB(&I);
3247 VAStartInstrumentationList.push_back(&I);
3248 Value *VAListTag = I.getArgOperand(0);
3249 Value *ShadowPtr = MSV.getShadowPtr(VAListTag, IRB.getInt8Ty(), IRB);
3250 // Unpoison the whole __va_list_tag.
3251 // FIXME: magic ABI constants (size of va_list).
3252 IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
3253 /* size */32, /* alignment */8, false);
3254 }
3255
visitVACopyInst__anon69a70b300211::VarArgAArch64Helper3256 void visitVACopyInst(VACopyInst &I) override {
3257 IRBuilder<> IRB(&I);
3258 Value *VAListTag = I.getArgOperand(0);
3259 Value *ShadowPtr = MSV.getShadowPtr(VAListTag, IRB.getInt8Ty(), IRB);
3260 // Unpoison the whole __va_list_tag.
3261 // FIXME: magic ABI constants (size of va_list).
3262 IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
3263 /* size */32, /* alignment */8, false);
3264 }
3265
3266 // Retrieve a va_list field of 'void*' size.
getVAField64__anon69a70b300211::VarArgAArch64Helper3267 Value* getVAField64(IRBuilder<> &IRB, Value *VAListTag, int offset) {
3268 Value *SaveAreaPtrPtr =
3269 IRB.CreateIntToPtr(
3270 IRB.CreateAdd(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
3271 ConstantInt::get(MS.IntptrTy, offset)),
3272 Type::getInt64PtrTy(*MS.C));
3273 return IRB.CreateLoad(SaveAreaPtrPtr);
3274 }
3275
3276 // Retrieve a va_list field of 'int' size.
getVAField32__anon69a70b300211::VarArgAArch64Helper3277 Value* getVAField32(IRBuilder<> &IRB, Value *VAListTag, int offset) {
3278 Value *SaveAreaPtr =
3279 IRB.CreateIntToPtr(
3280 IRB.CreateAdd(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
3281 ConstantInt::get(MS.IntptrTy, offset)),
3282 Type::getInt32PtrTy(*MS.C));
3283 Value *SaveArea32 = IRB.CreateLoad(SaveAreaPtr);
3284 return IRB.CreateSExt(SaveArea32, MS.IntptrTy);
3285 }
3286
finalizeInstrumentation__anon69a70b300211::VarArgAArch64Helper3287 void finalizeInstrumentation() override {
3288 assert(!VAArgOverflowSize && !VAArgTLSCopy &&
3289 "finalizeInstrumentation called twice");
3290 if (!VAStartInstrumentationList.empty()) {
3291 // If there is a va_start in this function, make a backup copy of
3292 // va_arg_tls somewhere in the function entry block.
3293 IRBuilder<> IRB(F.getEntryBlock().getFirstNonPHI());
3294 VAArgOverflowSize = IRB.CreateLoad(MS.VAArgOverflowSizeTLS);
3295 Value *CopySize =
3296 IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, AArch64VAEndOffset),
3297 VAArgOverflowSize);
3298 VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
3299 IRB.CreateMemCpy(VAArgTLSCopy, MS.VAArgTLS, CopySize, 8);
3300 }
3301
3302 Value *GrArgSize = ConstantInt::get(MS.IntptrTy, kAArch64GrArgSize);
3303 Value *VrArgSize = ConstantInt::get(MS.IntptrTy, kAArch64VrArgSize);
3304
3305 // Instrument va_start, copy va_list shadow from the backup copy of
3306 // the TLS contents.
3307 for (size_t i = 0, n = VAStartInstrumentationList.size(); i < n; i++) {
3308 CallInst *OrigInst = VAStartInstrumentationList[i];
3309 IRBuilder<> IRB(OrigInst->getNextNode());
3310
3311 Value *VAListTag = OrigInst->getArgOperand(0);
3312
3313 // The variadic ABI for AArch64 creates two areas to save the incoming
3314 // argument registers (one for 64-bit general register xn-x7 and another
3315 // for 128-bit FP/SIMD vn-v7).
3316 // We need then to propagate the shadow arguments on both regions
3317 // 'va::__gr_top + va::__gr_offs' and 'va::__vr_top + va::__vr_offs'.
3318 // The remaning arguments are saved on shadow for 'va::stack'.
3319 // One caveat is it requires only to propagate the non-named arguments,
3320 // however on the call site instrumentation 'all' the arguments are
3321 // saved. So to copy the shadow values from the va_arg TLS array
3322 // we need to adjust the offset for both GR and VR fields based on
3323 // the __{gr,vr}_offs value (since they are stores based on incoming
3324 // named arguments).
3325
3326 // Read the stack pointer from the va_list.
3327 Value *StackSaveAreaPtr = getVAField64(IRB, VAListTag, 0);
3328
3329 // Read both the __gr_top and __gr_off and add them up.
3330 Value *GrTopSaveAreaPtr = getVAField64(IRB, VAListTag, 8);
3331 Value *GrOffSaveArea = getVAField32(IRB, VAListTag, 24);
3332
3333 Value *GrRegSaveAreaPtr = IRB.CreateAdd(GrTopSaveAreaPtr, GrOffSaveArea);
3334
3335 // Read both the __vr_top and __vr_off and add them up.
3336 Value *VrTopSaveAreaPtr = getVAField64(IRB, VAListTag, 16);
3337 Value *VrOffSaveArea = getVAField32(IRB, VAListTag, 28);
3338
3339 Value *VrRegSaveAreaPtr = IRB.CreateAdd(VrTopSaveAreaPtr, VrOffSaveArea);
3340
3341 // It does not know how many named arguments is being used and, on the
3342 // callsite all the arguments were saved. Since __gr_off is defined as
3343 // '0 - ((8 - named_gr) * 8)', the idea is to just propagate the variadic
3344 // argument by ignoring the bytes of shadow from named arguments.
3345 Value *GrRegSaveAreaShadowPtrOff =
3346 IRB.CreateAdd(GrArgSize, GrOffSaveArea);
3347
3348 Value *GrRegSaveAreaShadowPtr =
3349 MSV.getShadowPtr(GrRegSaveAreaPtr, IRB.getInt8Ty(), IRB);
3350
3351 Value *GrSrcPtr = IRB.CreateInBoundsGEP(IRB.getInt8Ty(), VAArgTLSCopy,
3352 GrRegSaveAreaShadowPtrOff);
3353 Value *GrCopySize = IRB.CreateSub(GrArgSize, GrRegSaveAreaShadowPtrOff);
3354
3355 IRB.CreateMemCpy(GrRegSaveAreaShadowPtr, GrSrcPtr, GrCopySize, 8);
3356
3357 // Again, but for FP/SIMD values.
3358 Value *VrRegSaveAreaShadowPtrOff =
3359 IRB.CreateAdd(VrArgSize, VrOffSaveArea);
3360
3361 Value *VrRegSaveAreaShadowPtr =
3362 MSV.getShadowPtr(VrRegSaveAreaPtr, IRB.getInt8Ty(), IRB);
3363
3364 Value *VrSrcPtr = IRB.CreateInBoundsGEP(
3365 IRB.getInt8Ty(),
3366 IRB.CreateInBoundsGEP(IRB.getInt8Ty(), VAArgTLSCopy,
3367 IRB.getInt32(AArch64VrBegOffset)),
3368 VrRegSaveAreaShadowPtrOff);
3369 Value *VrCopySize = IRB.CreateSub(VrArgSize, VrRegSaveAreaShadowPtrOff);
3370
3371 IRB.CreateMemCpy(VrRegSaveAreaShadowPtr, VrSrcPtr, VrCopySize, 8);
3372
3373 // And finally for remaining arguments.
3374 Value *StackSaveAreaShadowPtr =
3375 MSV.getShadowPtr(StackSaveAreaPtr, IRB.getInt8Ty(), IRB);
3376
3377 Value *StackSrcPtr =
3378 IRB.CreateInBoundsGEP(IRB.getInt8Ty(), VAArgTLSCopy,
3379 IRB.getInt32(AArch64VAEndOffset));
3380
3381 IRB.CreateMemCpy(StackSaveAreaShadowPtr, StackSrcPtr,
3382 VAArgOverflowSize, 16);
3383 }
3384 }
3385 };
3386
3387 /// \brief PowerPC64-specific implementation of VarArgHelper.
3388 struct VarArgPowerPC64Helper : public VarArgHelper {
3389 Function &F;
3390 MemorySanitizer &MS;
3391 MemorySanitizerVisitor &MSV;
3392 Value *VAArgTLSCopy;
3393 Value *VAArgSize;
3394
3395 SmallVector<CallInst*, 16> VAStartInstrumentationList;
3396
VarArgPowerPC64Helper__anon69a70b300211::VarArgPowerPC64Helper3397 VarArgPowerPC64Helper(Function &F, MemorySanitizer &MS,
3398 MemorySanitizerVisitor &MSV)
3399 : F(F), MS(MS), MSV(MSV), VAArgTLSCopy(nullptr),
3400 VAArgSize(nullptr) {}
3401
visitCallSite__anon69a70b300211::VarArgPowerPC64Helper3402 void visitCallSite(CallSite &CS, IRBuilder<> &IRB) override {
3403 // For PowerPC, we need to deal with alignment of stack arguments -
3404 // they are mostly aligned to 8 bytes, but vectors and i128 arrays
3405 // are aligned to 16 bytes, byvals can be aligned to 8 or 16 bytes,
3406 // and QPX vectors are aligned to 32 bytes. For that reason, we
3407 // compute current offset from stack pointer (which is always properly
3408 // aligned), and offset for the first vararg, then subtract them.
3409 unsigned VAArgBase;
3410 llvm::Triple TargetTriple(F.getParent()->getTargetTriple());
3411 // Parameter save area starts at 48 bytes from frame pointer for ABIv1,
3412 // and 32 bytes for ABIv2. This is usually determined by target
3413 // endianness, but in theory could be overriden by function attribute.
3414 // For simplicity, we ignore it here (it'd only matter for QPX vectors).
3415 if (TargetTriple.getArch() == llvm::Triple::ppc64)
3416 VAArgBase = 48;
3417 else
3418 VAArgBase = 32;
3419 unsigned VAArgOffset = VAArgBase;
3420 const DataLayout &DL = F.getParent()->getDataLayout();
3421 for (CallSite::arg_iterator ArgIt = CS.arg_begin(), End = CS.arg_end();
3422 ArgIt != End; ++ArgIt) {
3423 Value *A = *ArgIt;
3424 unsigned ArgNo = CS.getArgumentNo(ArgIt);
3425 bool IsFixed = ArgNo < CS.getFunctionType()->getNumParams();
3426 bool IsByVal = CS.paramHasAttr(ArgNo + 1, Attribute::ByVal);
3427 if (IsByVal) {
3428 assert(A->getType()->isPointerTy());
3429 Type *RealTy = A->getType()->getPointerElementType();
3430 uint64_t ArgSize = DL.getTypeAllocSize(RealTy);
3431 uint64_t ArgAlign = CS.getParamAlignment(ArgNo + 1);
3432 if (ArgAlign < 8)
3433 ArgAlign = 8;
3434 VAArgOffset = alignTo(VAArgOffset, ArgAlign);
3435 if (!IsFixed) {
3436 Value *Base = getShadowPtrForVAArgument(RealTy, IRB,
3437 VAArgOffset - VAArgBase);
3438 IRB.CreateMemCpy(Base, MSV.getShadowPtr(A, IRB.getInt8Ty(), IRB),
3439 ArgSize, kShadowTLSAlignment);
3440 }
3441 VAArgOffset += alignTo(ArgSize, 8);
3442 } else {
3443 Value *Base;
3444 uint64_t ArgSize = DL.getTypeAllocSize(A->getType());
3445 uint64_t ArgAlign = 8;
3446 if (A->getType()->isArrayTy()) {
3447 // Arrays are aligned to element size, except for long double
3448 // arrays, which are aligned to 8 bytes.
3449 Type *ElementTy = A->getType()->getArrayElementType();
3450 if (!ElementTy->isPPC_FP128Ty())
3451 ArgAlign = DL.getTypeAllocSize(ElementTy);
3452 } else if (A->getType()->isVectorTy()) {
3453 // Vectors are naturally aligned.
3454 ArgAlign = DL.getTypeAllocSize(A->getType());
3455 }
3456 if (ArgAlign < 8)
3457 ArgAlign = 8;
3458 VAArgOffset = alignTo(VAArgOffset, ArgAlign);
3459 if (DL.isBigEndian()) {
3460 // Adjusting the shadow for argument with size < 8 to match the placement
3461 // of bits in big endian system
3462 if (ArgSize < 8)
3463 VAArgOffset += (8 - ArgSize);
3464 }
3465 if (!IsFixed) {
3466 Base = getShadowPtrForVAArgument(A->getType(), IRB,
3467 VAArgOffset - VAArgBase);
3468 IRB.CreateAlignedStore(MSV.getShadow(A), Base, kShadowTLSAlignment);
3469 }
3470 VAArgOffset += ArgSize;
3471 VAArgOffset = alignTo(VAArgOffset, 8);
3472 }
3473 if (IsFixed)
3474 VAArgBase = VAArgOffset;
3475 }
3476
3477 Constant *TotalVAArgSize = ConstantInt::get(IRB.getInt64Ty(),
3478 VAArgOffset - VAArgBase);
3479 // Here using VAArgOverflowSizeTLS as VAArgSizeTLS to avoid creation of
3480 // a new class member i.e. it is the total size of all VarArgs.
3481 IRB.CreateStore(TotalVAArgSize, MS.VAArgOverflowSizeTLS);
3482 }
3483
3484 /// \brief Compute the shadow address for a given va_arg.
getShadowPtrForVAArgument__anon69a70b300211::VarArgPowerPC64Helper3485 Value *getShadowPtrForVAArgument(Type *Ty, IRBuilder<> &IRB,
3486 int ArgOffset) {
3487 Value *Base = IRB.CreatePointerCast(MS.VAArgTLS, MS.IntptrTy);
3488 Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
3489 return IRB.CreateIntToPtr(Base, PointerType::get(MSV.getShadowTy(Ty), 0),
3490 "_msarg");
3491 }
3492
visitVAStartInst__anon69a70b300211::VarArgPowerPC64Helper3493 void visitVAStartInst(VAStartInst &I) override {
3494 IRBuilder<> IRB(&I);
3495 VAStartInstrumentationList.push_back(&I);
3496 Value *VAListTag = I.getArgOperand(0);
3497 Value *ShadowPtr = MSV.getShadowPtr(VAListTag, IRB.getInt8Ty(), IRB);
3498 IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
3499 /* size */8, /* alignment */8, false);
3500 }
3501
visitVACopyInst__anon69a70b300211::VarArgPowerPC64Helper3502 void visitVACopyInst(VACopyInst &I) override {
3503 IRBuilder<> IRB(&I);
3504 Value *VAListTag = I.getArgOperand(0);
3505 Value *ShadowPtr = MSV.getShadowPtr(VAListTag, IRB.getInt8Ty(), IRB);
3506 // Unpoison the whole __va_list_tag.
3507 // FIXME: magic ABI constants.
3508 IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
3509 /* size */8, /* alignment */8, false);
3510 }
3511
finalizeInstrumentation__anon69a70b300211::VarArgPowerPC64Helper3512 void finalizeInstrumentation() override {
3513 assert(!VAArgSize && !VAArgTLSCopy &&
3514 "finalizeInstrumentation called twice");
3515 IRBuilder<> IRB(F.getEntryBlock().getFirstNonPHI());
3516 VAArgSize = IRB.CreateLoad(MS.VAArgOverflowSizeTLS);
3517 Value *CopySize = IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, 0),
3518 VAArgSize);
3519
3520 if (!VAStartInstrumentationList.empty()) {
3521 // If there is a va_start in this function, make a backup copy of
3522 // va_arg_tls somewhere in the function entry block.
3523 VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
3524 IRB.CreateMemCpy(VAArgTLSCopy, MS.VAArgTLS, CopySize, 8);
3525 }
3526
3527 // Instrument va_start.
3528 // Copy va_list shadow from the backup copy of the TLS contents.
3529 for (size_t i = 0, n = VAStartInstrumentationList.size(); i < n; i++) {
3530 CallInst *OrigInst = VAStartInstrumentationList[i];
3531 IRBuilder<> IRB(OrigInst->getNextNode());
3532 Value *VAListTag = OrigInst->getArgOperand(0);
3533 Value *RegSaveAreaPtrPtr =
3534 IRB.CreateIntToPtr(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
3535 Type::getInt64PtrTy(*MS.C));
3536 Value *RegSaveAreaPtr = IRB.CreateLoad(RegSaveAreaPtrPtr);
3537 Value *RegSaveAreaShadowPtr =
3538 MSV.getShadowPtr(RegSaveAreaPtr, IRB.getInt8Ty(), IRB);
3539 IRB.CreateMemCpy(RegSaveAreaShadowPtr, VAArgTLSCopy, CopySize, 8);
3540 }
3541 }
3542 };
3543
3544 /// \brief A no-op implementation of VarArgHelper.
3545 struct VarArgNoOpHelper : public VarArgHelper {
VarArgNoOpHelper__anon69a70b300211::VarArgNoOpHelper3546 VarArgNoOpHelper(Function &F, MemorySanitizer &MS,
3547 MemorySanitizerVisitor &MSV) {}
3548
visitCallSite__anon69a70b300211::VarArgNoOpHelper3549 void visitCallSite(CallSite &CS, IRBuilder<> &IRB) override {}
3550
visitVAStartInst__anon69a70b300211::VarArgNoOpHelper3551 void visitVAStartInst(VAStartInst &I) override {}
3552
visitVACopyInst__anon69a70b300211::VarArgNoOpHelper3553 void visitVACopyInst(VACopyInst &I) override {}
3554
finalizeInstrumentation__anon69a70b300211::VarArgNoOpHelper3555 void finalizeInstrumentation() override {}
3556 };
3557
CreateVarArgHelper(Function & Func,MemorySanitizer & Msan,MemorySanitizerVisitor & Visitor)3558 VarArgHelper *CreateVarArgHelper(Function &Func, MemorySanitizer &Msan,
3559 MemorySanitizerVisitor &Visitor) {
3560 // VarArg handling is only implemented on AMD64. False positives are possible
3561 // on other platforms.
3562 llvm::Triple TargetTriple(Func.getParent()->getTargetTriple());
3563 if (TargetTriple.getArch() == llvm::Triple::x86_64)
3564 return new VarArgAMD64Helper(Func, Msan, Visitor);
3565 else if (TargetTriple.getArch() == llvm::Triple::mips64 ||
3566 TargetTriple.getArch() == llvm::Triple::mips64el)
3567 return new VarArgMIPS64Helper(Func, Msan, Visitor);
3568 else if (TargetTriple.getArch() == llvm::Triple::aarch64)
3569 return new VarArgAArch64Helper(Func, Msan, Visitor);
3570 else if (TargetTriple.getArch() == llvm::Triple::ppc64 ||
3571 TargetTriple.getArch() == llvm::Triple::ppc64le)
3572 return new VarArgPowerPC64Helper(Func, Msan, Visitor);
3573 else
3574 return new VarArgNoOpHelper(Func, Msan, Visitor);
3575 }
3576
3577 } // anonymous namespace
3578
runOnFunction(Function & F)3579 bool MemorySanitizer::runOnFunction(Function &F) {
3580 if (&F == MsanCtorFunction)
3581 return false;
3582 MemorySanitizerVisitor Visitor(F, *this);
3583
3584 // Clear out readonly/readnone attributes.
3585 AttrBuilder B;
3586 B.addAttribute(Attribute::ReadOnly)
3587 .addAttribute(Attribute::ReadNone);
3588 F.removeAttributes(AttributeSet::FunctionIndex,
3589 AttributeSet::get(F.getContext(),
3590 AttributeSet::FunctionIndex, B));
3591
3592 return Visitor.runOnFunction();
3593 }
3594