1 /*===- InstrProfilingFile.c - Write instrumentation to a file -------------===*\
2 |*
3 |* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 |* See https://llvm.org/LICENSE.txt for license information.
5 |* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 |*
7 \*===----------------------------------------------------------------------===*/
8
9 #if !defined(__Fuchsia__)
10
11 #include <errno.h>
12 #include <stdio.h>
13 #include <stdlib.h>
14 #include <string.h>
15 #ifdef _MSC_VER
16 /* For _alloca. */
17 #include <malloc.h>
18 #endif
19 #if defined(_WIN32)
20 #include "WindowsMMap.h"
21 /* For _chsize_s */
22 #include <io.h>
23 #include <process.h>
24 #else
25 #include <sys/file.h>
26 #include <sys/mman.h>
27 #include <unistd.h>
28 #if defined(__linux__)
29 #include <sys/types.h>
30 #endif
31 #endif
32
33 #include "InstrProfiling.h"
34 #include "InstrProfilingInternal.h"
35 #include "InstrProfilingPort.h"
36 #include "InstrProfilingUtil.h"
37
38 /* From where is profile name specified.
39 * The order the enumerators define their
40 * precedence. Re-order them may lead to
41 * runtime behavior change. */
42 typedef enum ProfileNameSpecifier {
43 PNS_unknown = 0,
44 PNS_default,
45 PNS_command_line,
46 PNS_environment,
47 PNS_runtime_api
48 } ProfileNameSpecifier;
49
getPNSStr(ProfileNameSpecifier PNS)50 static const char *getPNSStr(ProfileNameSpecifier PNS) {
51 switch (PNS) {
52 case PNS_default:
53 return "default setting";
54 case PNS_command_line:
55 return "command line";
56 case PNS_environment:
57 return "environment variable";
58 case PNS_runtime_api:
59 return "runtime API";
60 default:
61 return "Unknown";
62 }
63 }
64
65 #define MAX_PID_SIZE 16
66 /* Data structure holding the result of parsed filename pattern. */
67 typedef struct lprofFilename {
68 /* File name string possibly with %p or %h specifiers. */
69 const char *FilenamePat;
70 /* A flag indicating if FilenamePat's memory is allocated
71 * by runtime. */
72 unsigned OwnsFilenamePat;
73 const char *ProfilePathPrefix;
74 char PidChars[MAX_PID_SIZE];
75 char *TmpDir;
76 char Hostname[COMPILER_RT_MAX_HOSTLEN];
77 unsigned NumPids;
78 unsigned NumHosts;
79 /* When in-process merging is enabled, this parameter specifies
80 * the total number of profile data files shared by all the processes
81 * spawned from the same binary. By default the value is 1. If merging
82 * is not enabled, its value should be 0. This parameter is specified
83 * by the %[0-9]m specifier. For instance %2m enables merging using
84 * 2 profile data files. %1m is equivalent to %m. Also %m specifier
85 * can only appear once at the end of the name pattern. */
86 unsigned MergePoolSize;
87 ProfileNameSpecifier PNS;
88 } lprofFilename;
89
90 static lprofFilename lprofCurFilename = {0, 0, 0, {0}, NULL,
91 {0}, 0, 0, 0, PNS_unknown};
92
93 static int ProfileMergeRequested = 0;
isProfileMergeRequested()94 static int isProfileMergeRequested() { return ProfileMergeRequested; }
setProfileMergeRequested(int EnableMerge)95 static void setProfileMergeRequested(int EnableMerge) {
96 ProfileMergeRequested = EnableMerge;
97 }
98
99 static FILE *ProfileFile = NULL;
getProfileFile()100 static FILE *getProfileFile() { return ProfileFile; }
setProfileFile(FILE * File)101 static void setProfileFile(FILE *File) { ProfileFile = File; }
102
__llvm_profile_set_file_object(FILE * File,int EnableMerge)103 COMPILER_RT_VISIBILITY void __llvm_profile_set_file_object(FILE *File,
104 int EnableMerge) {
105 if (__llvm_profile_is_continuous_mode_enabled()) {
106 PROF_WARN("__llvm_profile_set_file_object(fd=%d) not supported, because "
107 "continuous sync mode (%%c) is enabled",
108 fileno(File));
109 return;
110 }
111 setProfileFile(File);
112 setProfileMergeRequested(EnableMerge);
113 }
114
115 static int getCurFilenameLength();
116 static const char *getCurFilename(char *FilenameBuf, int ForceUseBuf);
doMerging()117 static unsigned doMerging() {
118 return lprofCurFilename.MergePoolSize || isProfileMergeRequested();
119 }
120
121 /* Return 1 if there is an error, otherwise return 0. */
fileWriter(ProfDataWriter * This,ProfDataIOVec * IOVecs,uint32_t NumIOVecs)122 static uint32_t fileWriter(ProfDataWriter *This, ProfDataIOVec *IOVecs,
123 uint32_t NumIOVecs) {
124 uint32_t I;
125 FILE *File = (FILE *)This->WriterCtx;
126 char Zeroes[sizeof(uint64_t)] = {0};
127 for (I = 0; I < NumIOVecs; I++) {
128 if (IOVecs[I].Data) {
129 if (fwrite(IOVecs[I].Data, IOVecs[I].ElmSize, IOVecs[I].NumElm, File) !=
130 IOVecs[I].NumElm)
131 return 1;
132 } else if (IOVecs[I].UseZeroPadding) {
133 size_t BytesToWrite = IOVecs[I].ElmSize * IOVecs[I].NumElm;
134 while (BytesToWrite > 0) {
135 size_t PartialWriteLen =
136 (sizeof(uint64_t) > BytesToWrite) ? BytesToWrite : sizeof(uint64_t);
137 if (fwrite(Zeroes, sizeof(uint8_t), PartialWriteLen, File) !=
138 PartialWriteLen) {
139 return 1;
140 }
141 BytesToWrite -= PartialWriteLen;
142 }
143 } else {
144 if (fseek(File, IOVecs[I].ElmSize * IOVecs[I].NumElm, SEEK_CUR) == -1)
145 return 1;
146 }
147 }
148 return 0;
149 }
150
151 /* TODO: make buffer size controllable by an internal option, and compiler can pass the size
152 to runtime via a variable. */
orderFileWriter(FILE * File,const uint32_t * DataStart)153 static uint32_t orderFileWriter(FILE *File, const uint32_t *DataStart) {
154 if (fwrite(DataStart, sizeof(uint32_t), INSTR_ORDER_FILE_BUFFER_SIZE, File) !=
155 INSTR_ORDER_FILE_BUFFER_SIZE)
156 return 1;
157 return 0;
158 }
159
initFileWriter(ProfDataWriter * This,FILE * File)160 static void initFileWriter(ProfDataWriter *This, FILE *File) {
161 This->Write = fileWriter;
162 This->WriterCtx = File;
163 }
164
165 COMPILER_RT_VISIBILITY ProfBufferIO *
lprofCreateBufferIOInternal(void * File,uint32_t BufferSz)166 lprofCreateBufferIOInternal(void *File, uint32_t BufferSz) {
167 FreeHook = &free;
168 DynamicBufferIOBuffer = (uint8_t *)calloc(BufferSz, 1);
169 VPBufferSize = BufferSz;
170 ProfDataWriter *fileWriter =
171 (ProfDataWriter *)calloc(sizeof(ProfDataWriter), 1);
172 initFileWriter(fileWriter, File);
173 ProfBufferIO *IO = lprofCreateBufferIO(fileWriter);
174 IO->OwnFileWriter = 1;
175 return IO;
176 }
177
setupIOBuffer()178 static void setupIOBuffer() {
179 const char *BufferSzStr = 0;
180 BufferSzStr = getenv("LLVM_VP_BUFFER_SIZE");
181 if (BufferSzStr && BufferSzStr[0]) {
182 VPBufferSize = atoi(BufferSzStr);
183 DynamicBufferIOBuffer = (uint8_t *)calloc(VPBufferSize, 1);
184 }
185 }
186
187 /* Get the size of the profile file. If there are any errors, print the
188 * message under the assumption that the profile is being read for merging
189 * purposes, and return -1. Otherwise return the file size in the inout param
190 * \p ProfileFileSize. */
getProfileFileSizeForMerging(FILE * ProfileFile,uint64_t * ProfileFileSize)191 static int getProfileFileSizeForMerging(FILE *ProfileFile,
192 uint64_t *ProfileFileSize) {
193 if (fseek(ProfileFile, 0L, SEEK_END) == -1) {
194 PROF_ERR("Unable to merge profile data, unable to get size: %s\n",
195 strerror(errno));
196 return -1;
197 }
198 *ProfileFileSize = ftell(ProfileFile);
199
200 /* Restore file offset. */
201 if (fseek(ProfileFile, 0L, SEEK_SET) == -1) {
202 PROF_ERR("Unable to merge profile data, unable to rewind: %s\n",
203 strerror(errno));
204 return -1;
205 }
206
207 if (*ProfileFileSize > 0 &&
208 *ProfileFileSize < sizeof(__llvm_profile_header)) {
209 PROF_WARN("Unable to merge profile data: %s\n",
210 "source profile file is too small.");
211 return -1;
212 }
213 return 0;
214 }
215
216 /* mmap() \p ProfileFile for profile merging purposes, assuming that an
217 * exclusive lock is held on the file and that \p ProfileFileSize is the
218 * length of the file. Return the mmap'd buffer in the inout variable
219 * \p ProfileBuffer. Returns -1 on failure. On success, the caller is
220 * responsible for unmapping the mmap'd buffer in \p ProfileBuffer. */
mmapProfileForMerging(FILE * ProfileFile,uint64_t ProfileFileSize,char ** ProfileBuffer)221 static int mmapProfileForMerging(FILE *ProfileFile, uint64_t ProfileFileSize,
222 char **ProfileBuffer) {
223 *ProfileBuffer = mmap(NULL, ProfileFileSize, PROT_READ, MAP_SHARED | MAP_FILE,
224 fileno(ProfileFile), 0);
225 if (*ProfileBuffer == MAP_FAILED) {
226 PROF_ERR("Unable to merge profile data, mmap failed: %s\n",
227 strerror(errno));
228 return -1;
229 }
230
231 if (__llvm_profile_check_compatibility(*ProfileBuffer, ProfileFileSize)) {
232 (void)munmap(*ProfileBuffer, ProfileFileSize);
233 PROF_WARN("Unable to merge profile data: %s\n",
234 "source profile file is not compatible.");
235 return -1;
236 }
237 return 0;
238 }
239
240 /* Read profile data in \c ProfileFile and merge with in-memory
241 profile counters. Returns -1 if there is fatal error, otheriwse
242 0 is returned. Returning 0 does not mean merge is actually
243 performed. If merge is actually done, *MergeDone is set to 1.
244 */
doProfileMerging(FILE * ProfileFile,int * MergeDone)245 static int doProfileMerging(FILE *ProfileFile, int *MergeDone) {
246 uint64_t ProfileFileSize;
247 char *ProfileBuffer;
248
249 /* Get the size of the profile on disk. */
250 if (getProfileFileSizeForMerging(ProfileFile, &ProfileFileSize) == -1)
251 return -1;
252
253 /* Nothing to merge. */
254 if (!ProfileFileSize)
255 return 0;
256
257 /* mmap() the profile and check that it is compatible with the data in
258 * the current image. */
259 if (mmapProfileForMerging(ProfileFile, ProfileFileSize, &ProfileBuffer) == -1)
260 return -1;
261
262 /* Now start merging */
263 __llvm_profile_merge_from_buffer(ProfileBuffer, ProfileFileSize);
264
265 // Truncate the file in case merging of value profile did not happend to
266 // prevent from leaving garbage data at the end of the profile file.
267 COMPILER_RT_FTRUNCATE(ProfileFile, __llvm_profile_get_size_for_buffer());
268
269 (void)munmap(ProfileBuffer, ProfileFileSize);
270 *MergeDone = 1;
271
272 return 0;
273 }
274
275 /* Create the directory holding the file, if needed. */
createProfileDir(const char * Filename)276 static void createProfileDir(const char *Filename) {
277 size_t Length = strlen(Filename);
278 if (lprofFindFirstDirSeparator(Filename)) {
279 char *Copy = (char *)COMPILER_RT_ALLOCA(Length + 1);
280 strncpy(Copy, Filename, Length + 1);
281 __llvm_profile_recursive_mkdir(Copy);
282 }
283 }
284
285 /* Open the profile data for merging. It opens the file in r+b mode with
286 * file locking. If the file has content which is compatible with the
287 * current process, it also reads in the profile data in the file and merge
288 * it with in-memory counters. After the profile data is merged in memory,
289 * the original profile data is truncated and gets ready for the profile
290 * dumper. With profile merging enabled, each executable as well as any of
291 * its instrumented shared libraries dump profile data into their own data file.
292 */
openFileForMerging(const char * ProfileFileName,int * MergeDone)293 static FILE *openFileForMerging(const char *ProfileFileName, int *MergeDone) {
294 FILE *ProfileFile = NULL;
295 int rc;
296
297 ProfileFile = getProfileFile();
298 if (ProfileFile) {
299 lprofLockFileHandle(ProfileFile);
300 } else {
301 createProfileDir(ProfileFileName);
302 ProfileFile = lprofOpenFileEx(ProfileFileName);
303 }
304 if (!ProfileFile)
305 return NULL;
306
307 rc = doProfileMerging(ProfileFile, MergeDone);
308 if (rc || (!*MergeDone && COMPILER_RT_FTRUNCATE(ProfileFile, 0L)) ||
309 fseek(ProfileFile, 0L, SEEK_SET) == -1) {
310 PROF_ERR("Profile Merging of file %s failed: %s\n", ProfileFileName,
311 strerror(errno));
312 fclose(ProfileFile);
313 return NULL;
314 }
315 return ProfileFile;
316 }
317
getFileObject(const char * OutputName)318 static FILE *getFileObject(const char *OutputName) {
319 FILE *File;
320 File = getProfileFile();
321 if (File != NULL) {
322 return File;
323 }
324
325 return fopen(OutputName, "ab");
326 }
327
328 /* Write profile data to file \c OutputName. */
writeFile(const char * OutputName)329 static int writeFile(const char *OutputName) {
330 int RetVal;
331 FILE *OutputFile;
332
333 int MergeDone = 0;
334 VPMergeHook = &lprofMergeValueProfData;
335 if (doMerging())
336 OutputFile = openFileForMerging(OutputName, &MergeDone);
337 else
338 OutputFile = getFileObject(OutputName);
339
340 if (!OutputFile)
341 return -1;
342
343 FreeHook = &free;
344 setupIOBuffer();
345 ProfDataWriter fileWriter;
346 initFileWriter(&fileWriter, OutputFile);
347 RetVal = lprofWriteData(&fileWriter, lprofGetVPDataReader(), MergeDone);
348
349 if (OutputFile == getProfileFile()) {
350 fflush(OutputFile);
351 if (doMerging()) {
352 lprofUnlockFileHandle(OutputFile);
353 }
354 } else {
355 fclose(OutputFile);
356 }
357
358 return RetVal;
359 }
360
361 /* Write order data to file \c OutputName. */
writeOrderFile(const char * OutputName)362 static int writeOrderFile(const char *OutputName) {
363 int RetVal;
364 FILE *OutputFile;
365
366 OutputFile = fopen(OutputName, "w");
367
368 if (!OutputFile) {
369 PROF_WARN("can't open file with mode ab: %s\n", OutputName);
370 return -1;
371 }
372
373 FreeHook = &free;
374 setupIOBuffer();
375 const uint32_t *DataBegin = __llvm_profile_begin_orderfile();
376 RetVal = orderFileWriter(OutputFile, DataBegin);
377
378 fclose(OutputFile);
379 return RetVal;
380 }
381
382 #define LPROF_INIT_ONCE_ENV "__LLVM_PROFILE_RT_INIT_ONCE"
383
truncateCurrentFile(void)384 static void truncateCurrentFile(void) {
385 const char *Filename;
386 char *FilenameBuf;
387 FILE *File;
388 int Length;
389
390 Length = getCurFilenameLength();
391 FilenameBuf = (char *)COMPILER_RT_ALLOCA(Length + 1);
392 Filename = getCurFilename(FilenameBuf, 0);
393 if (!Filename)
394 return;
395
396 /* Only create the profile directory and truncate an existing profile once.
397 * In continuous mode, this is necessary, as the profile is written-to by the
398 * runtime initializer. */
399 int initialized = getenv(LPROF_INIT_ONCE_ENV) != NULL;
400 if (initialized)
401 return;
402 #if defined(_WIN32)
403 _putenv(LPROF_INIT_ONCE_ENV "=" LPROF_INIT_ONCE_ENV);
404 #else
405 setenv(LPROF_INIT_ONCE_ENV, LPROF_INIT_ONCE_ENV, 1);
406 #endif
407
408 /* Create the profile dir (even if online merging is enabled), so that
409 * the profile file can be set up if continuous mode is enabled. */
410 createProfileDir(Filename);
411
412 /* By pass file truncation to allow online raw profile merging. */
413 if (lprofCurFilename.MergePoolSize)
414 return;
415
416 /* Truncate the file. Later we'll reopen and append. */
417 File = fopen(Filename, "w");
418 if (!File)
419 return;
420 fclose(File);
421 }
422
423 #if !defined(__Fuchsia__) && !defined(_WIN32)
assertIsZero(int * i)424 static void assertIsZero(int *i) {
425 if (*i)
426 PROF_WARN("Expected flag to be 0, but got: %d\n", *i);
427 }
428
429 /* Write a partial profile to \p Filename, which is required to be backed by
430 * the open file object \p File. */
writeProfileWithFileObject(const char * Filename,FILE * File)431 static int writeProfileWithFileObject(const char *Filename, FILE *File) {
432 setProfileFile(File);
433 int rc = writeFile(Filename);
434 if (rc)
435 PROF_ERR("Failed to write file \"%s\": %s\n", Filename, strerror(errno));
436 setProfileFile(NULL);
437 return rc;
438 }
439
440 /* Unlock the profile \p File and clear the unlock flag. */
unlockProfile(int * ProfileRequiresUnlock,FILE * File)441 static void unlockProfile(int *ProfileRequiresUnlock, FILE *File) {
442 if (!*ProfileRequiresUnlock) {
443 PROF_WARN("%s", "Expected to require profile unlock\n");
444 }
445 lprofUnlockFileHandle(File);
446 *ProfileRequiresUnlock = 0;
447 }
448 #endif // !defined(__Fuchsia__) && !defined(_WIN32)
449
writeMMappedFile(FILE * OutputFile,char ** Profile)450 static int writeMMappedFile(FILE *OutputFile, char **Profile) {
451 if (!OutputFile)
452 return -1;
453
454 /* Write the data into a file. */
455 setupIOBuffer();
456 ProfDataWriter fileWriter;
457 initFileWriter(&fileWriter, OutputFile);
458 if (lprofWriteData(&fileWriter, NULL, 0)) {
459 PROF_ERR("Failed to write profile: %s\n", strerror(errno));
460 return -1;
461 }
462 fflush(OutputFile);
463
464 /* Get the file size. */
465 uint64_t FileSize = ftell(OutputFile);
466
467 /* Map the profile. */
468 *Profile = (char *)mmap(
469 NULL, FileSize, PROT_READ | PROT_WRITE, MAP_SHARED, fileno(OutputFile), 0);
470 if (*Profile == MAP_FAILED) {
471 PROF_ERR("Unable to mmap profile: %s\n", strerror(errno));
472 return -1;
473 }
474
475 return 0;
476 }
477
relocateCounters(void)478 static void relocateCounters(void) {
479 if (!__llvm_profile_is_continuous_mode_enabled() ||
480 !lprofRuntimeCounterRelocation())
481 return;
482
483 /* Get the sizes of various profile data sections. Taken from
484 * __llvm_profile_get_size_for_buffer(). */
485 const __llvm_profile_data *DataBegin = __llvm_profile_begin_data();
486 const __llvm_profile_data *DataEnd = __llvm_profile_end_data();
487 uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd);
488 const uint64_t CountersOffset = sizeof(__llvm_profile_header) +
489 (DataSize * sizeof(__llvm_profile_data));
490
491 int Length = getCurFilenameLength();
492 char *FilenameBuf = (char *)COMPILER_RT_ALLOCA(Length + 1);
493 const char *Filename = getCurFilename(FilenameBuf, 0);
494 if (!Filename)
495 return;
496
497 FILE *File = NULL;
498 char *Profile = NULL;
499
500 if (!doMerging()) {
501 File = fopen(Filename, "w+b");
502 if (!File)
503 return;
504
505 if (writeMMappedFile(File, &Profile) == -1) {
506 fclose(File);
507 return;
508 }
509 } else {
510 File = lprofOpenFileEx(Filename);
511 if (!File)
512 return;
513
514 uint64_t ProfileFileSize = 0;
515 if (getProfileFileSizeForMerging(File, &ProfileFileSize) == -1) {
516 lprofUnlockFileHandle(File);
517 fclose(File);
518 return;
519 }
520
521 if (!ProfileFileSize) {
522 if (writeMMappedFile(File, &Profile) == -1) {
523 fclose(File);
524 return;
525 }
526 } else {
527 /* The merged profile has a non-zero length. Check that it is compatible
528 * with the data in this process. */
529 if (mmapProfileForMerging(File, ProfileFileSize, &Profile) == -1) {
530 fclose(File);
531 return;
532 }
533 }
534
535 lprofUnlockFileHandle(File);
536 }
537
538 /* Update the profile fields based on the current mapping. */
539 __llvm_profile_counter_bias = (intptr_t)Profile -
540 (uintptr_t)__llvm_profile_begin_counters() + CountersOffset;
541 }
542
initializeProfileForContinuousMode(void)543 static void initializeProfileForContinuousMode(void) {
544 if (!__llvm_profile_is_continuous_mode_enabled())
545 return;
546
547 #if defined(__Fuchsia__) || defined(_WIN32)
548 PROF_ERR("%s\n", "Continuous mode not yet supported on Fuchsia or Windows.");
549 #else // defined(__Fuchsia__) || defined(_WIN32)
550 /* Get the sizes of various profile data sections. Taken from
551 * __llvm_profile_get_size_for_buffer(). */
552 const __llvm_profile_data *DataBegin = __llvm_profile_begin_data();
553 const __llvm_profile_data *DataEnd = __llvm_profile_end_data();
554 const uint64_t *CountersBegin = __llvm_profile_begin_counters();
555 const uint64_t *CountersEnd = __llvm_profile_end_counters();
556 const char *NamesBegin = __llvm_profile_begin_names();
557 const char *NamesEnd = __llvm_profile_end_names();
558 const uint64_t NamesSize = (NamesEnd - NamesBegin) * sizeof(char);
559 uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd);
560 uint64_t CountersSize = CountersEnd - CountersBegin;
561
562 /* Check that the counter and data sections in this image are page-aligned. */
563 unsigned PageSize = getpagesize();
564 if ((intptr_t)CountersBegin % PageSize != 0) {
565 PROF_ERR("Counters section not page-aligned (start = %p, pagesz = %u).\n",
566 CountersBegin, PageSize);
567 return;
568 }
569 if ((intptr_t)DataBegin % PageSize != 0) {
570 PROF_ERR("Data section not page-aligned (start = %p, pagesz = %u).\n",
571 DataBegin, PageSize);
572 return;
573 }
574
575 int Length = getCurFilenameLength();
576 char *FilenameBuf = (char *)COMPILER_RT_ALLOCA(Length + 1);
577 const char *Filename = getCurFilename(FilenameBuf, 0);
578 if (!Filename)
579 return;
580
581 FILE *File = NULL;
582 off_t CurrentFileOffset = 0;
583 off_t OffsetModPage = 0;
584
585 /* Whether an exclusive lock on the profile must be dropped after init.
586 * Use a cleanup to warn if the unlock does not occur. */
587 COMPILER_RT_CLEANUP(assertIsZero) int ProfileRequiresUnlock = 0;
588
589 if (!doMerging()) {
590 /* We are not merging profiles, so open the raw profile in append mode. */
591 File = fopen(Filename, "a+b");
592 if (!File)
593 return;
594
595 /* Check that the offset within the file is page-aligned. */
596 CurrentFileOffset = ftello(File);
597 OffsetModPage = CurrentFileOffset % PageSize;
598 if (OffsetModPage != 0) {
599 PROF_ERR("Continuous counter sync mode is enabled, but raw profile is not"
600 "page-aligned. CurrentFileOffset = %" PRIu64 ", pagesz = %u.\n",
601 (uint64_t)CurrentFileOffset, PageSize);
602 return;
603 }
604
605 /* Grow the profile so that mmap() can succeed. Leak the file handle, as
606 * the file should stay open. */
607 if (writeProfileWithFileObject(Filename, File) != 0)
608 return;
609 } else {
610 /* We are merging profiles. Map the counter section as shared memory into
611 * the profile, i.e. into each participating process. An increment in one
612 * process should be visible to every other process with the same counter
613 * section mapped. */
614 File = lprofOpenFileEx(Filename);
615 if (!File)
616 return;
617
618 ProfileRequiresUnlock = 1;
619
620 uint64_t ProfileFileSize;
621 if (getProfileFileSizeForMerging(File, &ProfileFileSize) == -1)
622 return unlockProfile(&ProfileRequiresUnlock, File);
623
624 if (ProfileFileSize == 0) {
625 /* Grow the profile so that mmap() can succeed. Leak the file handle, as
626 * the file should stay open. */
627 if (writeProfileWithFileObject(Filename, File) != 0)
628 return unlockProfile(&ProfileRequiresUnlock, File);
629 } else {
630 /* The merged profile has a non-zero length. Check that it is compatible
631 * with the data in this process. */
632 char *ProfileBuffer;
633 if (mmapProfileForMerging(File, ProfileFileSize, &ProfileBuffer) == -1 ||
634 munmap(ProfileBuffer, ProfileFileSize) == -1)
635 return unlockProfile(&ProfileRequiresUnlock, File);
636 }
637 }
638
639 int Fileno = fileno(File);
640
641 /* Determine how much padding is needed before/after the counters and after
642 * the names. */
643 uint64_t PaddingBytesBeforeCounters, PaddingBytesAfterCounters,
644 PaddingBytesAfterNames;
645 __llvm_profile_get_padding_sizes_for_counters(
646 DataSize, CountersSize, NamesSize, &PaddingBytesBeforeCounters,
647 &PaddingBytesAfterCounters, &PaddingBytesAfterNames);
648
649 uint64_t PageAlignedCountersLength =
650 (CountersSize * sizeof(uint64_t)) + PaddingBytesAfterCounters;
651 uint64_t FileOffsetToCounters =
652 CurrentFileOffset + sizeof(__llvm_profile_header) +
653 (DataSize * sizeof(__llvm_profile_data)) + PaddingBytesBeforeCounters;
654
655 uint64_t *CounterMmap = (uint64_t *)mmap(
656 (void *)CountersBegin, PageAlignedCountersLength, PROT_READ | PROT_WRITE,
657 MAP_FIXED | MAP_SHARED, Fileno, FileOffsetToCounters);
658 if (CounterMmap != CountersBegin) {
659 PROF_ERR(
660 "Continuous counter sync mode is enabled, but mmap() failed (%s).\n"
661 " - CountersBegin: %p\n"
662 " - PageAlignedCountersLength: %" PRIu64 "\n"
663 " - Fileno: %d\n"
664 " - FileOffsetToCounters: %" PRIu64 "\n",
665 strerror(errno), CountersBegin, PageAlignedCountersLength, Fileno,
666 FileOffsetToCounters);
667 }
668
669 if (ProfileRequiresUnlock)
670 unlockProfile(&ProfileRequiresUnlock, File);
671 #endif // defined(__Fuchsia__) || defined(_WIN32)
672 }
673
674 static const char *DefaultProfileName = "default.profraw";
resetFilenameToDefault(void)675 static void resetFilenameToDefault(void) {
676 if (lprofCurFilename.FilenamePat && lprofCurFilename.OwnsFilenamePat) {
677 free((void *)lprofCurFilename.FilenamePat);
678 }
679 memset(&lprofCurFilename, 0, sizeof(lprofCurFilename));
680 lprofCurFilename.FilenamePat = DefaultProfileName;
681 lprofCurFilename.PNS = PNS_default;
682 }
683
getMergePoolSize(const char * FilenamePat,int * I)684 static unsigned getMergePoolSize(const char *FilenamePat, int *I) {
685 unsigned J = 0, Num = 0;
686 for (;; ++J) {
687 char C = FilenamePat[*I + J];
688 if (C == 'm') {
689 *I += J;
690 return Num ? Num : 1;
691 }
692 if (C < '0' || C > '9')
693 break;
694 Num = Num * 10 + C - '0';
695
696 /* If FilenamePat[*I+J] is between '0' and '9', the next byte is guaranteed
697 * to be in-bound as the string is null terminated. */
698 }
699 return 0;
700 }
701
702 /* Parses the pattern string \p FilenamePat and stores the result to
703 * lprofcurFilename structure. */
parseFilenamePattern(const char * FilenamePat,unsigned CopyFilenamePat)704 static int parseFilenamePattern(const char *FilenamePat,
705 unsigned CopyFilenamePat) {
706 int NumPids = 0, NumHosts = 0, I;
707 char *PidChars = &lprofCurFilename.PidChars[0];
708 char *Hostname = &lprofCurFilename.Hostname[0];
709 int MergingEnabled = 0;
710
711 /* Clean up cached prefix and filename. */
712 if (lprofCurFilename.ProfilePathPrefix)
713 free((void *)lprofCurFilename.ProfilePathPrefix);
714
715 if (lprofCurFilename.FilenamePat && lprofCurFilename.OwnsFilenamePat) {
716 free((void *)lprofCurFilename.FilenamePat);
717 }
718
719 memset(&lprofCurFilename, 0, sizeof(lprofCurFilename));
720
721 if (!CopyFilenamePat)
722 lprofCurFilename.FilenamePat = FilenamePat;
723 else {
724 lprofCurFilename.FilenamePat = strdup(FilenamePat);
725 lprofCurFilename.OwnsFilenamePat = 1;
726 }
727 /* Check the filename for "%p", which indicates a pid-substitution. */
728 for (I = 0; FilenamePat[I]; ++I)
729 if (FilenamePat[I] == '%') {
730 if (FilenamePat[++I] == 'p') {
731 if (!NumPids++) {
732 if (snprintf(PidChars, MAX_PID_SIZE, "%ld", (long)getpid()) <= 0) {
733 PROF_WARN("Unable to get pid for filename pattern %s. Using the "
734 "default name.",
735 FilenamePat);
736 return -1;
737 }
738 }
739 } else if (FilenamePat[I] == 'h') {
740 if (!NumHosts++)
741 if (COMPILER_RT_GETHOSTNAME(Hostname, COMPILER_RT_MAX_HOSTLEN)) {
742 PROF_WARN("Unable to get hostname for filename pattern %s. Using "
743 "the default name.",
744 FilenamePat);
745 return -1;
746 }
747 } else if (FilenamePat[I] == 't') {
748 lprofCurFilename.TmpDir = getenv("TMPDIR");
749 if (!lprofCurFilename.TmpDir) {
750 PROF_WARN("Unable to get the TMPDIR environment variable, referenced "
751 "in %s. Using the default path.",
752 FilenamePat);
753 return -1;
754 }
755 } else if (FilenamePat[I] == 'c') {
756 if (__llvm_profile_is_continuous_mode_enabled()) {
757 PROF_WARN("%%c specifier can only be specified once in %s.\n",
758 FilenamePat);
759 return -1;
760 }
761
762 __llvm_profile_set_page_size(getpagesize());
763 __llvm_profile_enable_continuous_mode();
764 I++; /* advance to 'c' */
765 } else {
766 unsigned MergePoolSize = getMergePoolSize(FilenamePat, &I);
767 if (!MergePoolSize)
768 continue;
769 if (MergingEnabled) {
770 PROF_WARN("%%m specifier can only be specified once in %s.\n",
771 FilenamePat);
772 return -1;
773 }
774 MergingEnabled = 1;
775 lprofCurFilename.MergePoolSize = MergePoolSize;
776 }
777 }
778
779 lprofCurFilename.NumPids = NumPids;
780 lprofCurFilename.NumHosts = NumHosts;
781 return 0;
782 }
783
parseAndSetFilename(const char * FilenamePat,ProfileNameSpecifier PNS,unsigned CopyFilenamePat)784 static void parseAndSetFilename(const char *FilenamePat,
785 ProfileNameSpecifier PNS,
786 unsigned CopyFilenamePat) {
787
788 const char *OldFilenamePat = lprofCurFilename.FilenamePat;
789 ProfileNameSpecifier OldPNS = lprofCurFilename.PNS;
790
791 /* The old profile name specifier takes precedence over the old one. */
792 if (PNS < OldPNS)
793 return;
794
795 if (!FilenamePat)
796 FilenamePat = DefaultProfileName;
797
798 if (OldFilenamePat && !strcmp(OldFilenamePat, FilenamePat)) {
799 lprofCurFilename.PNS = PNS;
800 return;
801 }
802
803 /* When PNS >= OldPNS, the last one wins. */
804 if (!FilenamePat || parseFilenamePattern(FilenamePat, CopyFilenamePat))
805 resetFilenameToDefault();
806 lprofCurFilename.PNS = PNS;
807
808 if (!OldFilenamePat) {
809 if (getenv("LLVM_PROFILE_VERBOSE"))
810 PROF_NOTE("Set profile file path to \"%s\" via %s.\n",
811 lprofCurFilename.FilenamePat, getPNSStr(PNS));
812 } else {
813 if (getenv("LLVM_PROFILE_VERBOSE"))
814 PROF_NOTE("Override old profile path \"%s\" via %s to \"%s\" via %s.\n",
815 OldFilenamePat, getPNSStr(OldPNS), lprofCurFilename.FilenamePat,
816 getPNSStr(PNS));
817 }
818
819 truncateCurrentFile();
820 if (__llvm_profile_is_continuous_mode_enabled()) {
821 if (lprofRuntimeCounterRelocation())
822 relocateCounters();
823 else
824 initializeProfileForContinuousMode();
825 }
826 }
827
828 /* Return buffer length that is required to store the current profile
829 * filename with PID and hostname substitutions. */
830 /* The length to hold uint64_t followed by 3 digits pool id including '_' */
831 #define SIGLEN 24
getCurFilenameLength()832 static int getCurFilenameLength() {
833 int Len;
834 if (!lprofCurFilename.FilenamePat || !lprofCurFilename.FilenamePat[0])
835 return 0;
836
837 if (!(lprofCurFilename.NumPids || lprofCurFilename.NumHosts ||
838 lprofCurFilename.TmpDir || lprofCurFilename.MergePoolSize))
839 return strlen(lprofCurFilename.FilenamePat);
840
841 Len = strlen(lprofCurFilename.FilenamePat) +
842 lprofCurFilename.NumPids * (strlen(lprofCurFilename.PidChars) - 2) +
843 lprofCurFilename.NumHosts * (strlen(lprofCurFilename.Hostname) - 2) +
844 (lprofCurFilename.TmpDir ? (strlen(lprofCurFilename.TmpDir) - 1) : 0);
845 if (lprofCurFilename.MergePoolSize)
846 Len += SIGLEN;
847 return Len;
848 }
849
850 /* Return the pointer to the current profile file name (after substituting
851 * PIDs and Hostnames in filename pattern. \p FilenameBuf is the buffer
852 * to store the resulting filename. If no substitution is needed, the
853 * current filename pattern string is directly returned, unless ForceUseBuf
854 * is enabled. */
getCurFilename(char * FilenameBuf,int ForceUseBuf)855 static const char *getCurFilename(char *FilenameBuf, int ForceUseBuf) {
856 int I, J, PidLength, HostNameLength, TmpDirLength, FilenamePatLength;
857 const char *FilenamePat = lprofCurFilename.FilenamePat;
858
859 if (!lprofCurFilename.FilenamePat || !lprofCurFilename.FilenamePat[0])
860 return 0;
861
862 if (!(lprofCurFilename.NumPids || lprofCurFilename.NumHosts ||
863 lprofCurFilename.TmpDir || lprofCurFilename.MergePoolSize ||
864 __llvm_profile_is_continuous_mode_enabled())) {
865 if (!ForceUseBuf)
866 return lprofCurFilename.FilenamePat;
867
868 FilenamePatLength = strlen(lprofCurFilename.FilenamePat);
869 memcpy(FilenameBuf, lprofCurFilename.FilenamePat, FilenamePatLength);
870 FilenameBuf[FilenamePatLength] = '\0';
871 return FilenameBuf;
872 }
873
874 PidLength = strlen(lprofCurFilename.PidChars);
875 HostNameLength = strlen(lprofCurFilename.Hostname);
876 TmpDirLength = lprofCurFilename.TmpDir ? strlen(lprofCurFilename.TmpDir) : 0;
877 /* Construct the new filename. */
878 for (I = 0, J = 0; FilenamePat[I]; ++I)
879 if (FilenamePat[I] == '%') {
880 if (FilenamePat[++I] == 'p') {
881 memcpy(FilenameBuf + J, lprofCurFilename.PidChars, PidLength);
882 J += PidLength;
883 } else if (FilenamePat[I] == 'h') {
884 memcpy(FilenameBuf + J, lprofCurFilename.Hostname, HostNameLength);
885 J += HostNameLength;
886 } else if (FilenamePat[I] == 't') {
887 memcpy(FilenameBuf + J, lprofCurFilename.TmpDir, TmpDirLength);
888 FilenameBuf[J + TmpDirLength] = DIR_SEPARATOR;
889 J += TmpDirLength + 1;
890 } else {
891 if (!getMergePoolSize(FilenamePat, &I))
892 continue;
893 char LoadModuleSignature[SIGLEN + 1];
894 int S;
895 int ProfilePoolId = getpid() % lprofCurFilename.MergePoolSize;
896 S = snprintf(LoadModuleSignature, SIGLEN + 1, "%" PRIu64 "_%d",
897 lprofGetLoadModuleSignature(), ProfilePoolId);
898 if (S == -1 || S > SIGLEN)
899 S = SIGLEN;
900 memcpy(FilenameBuf + J, LoadModuleSignature, S);
901 J += S;
902 }
903 /* Drop any unknown substitutions. */
904 } else
905 FilenameBuf[J++] = FilenamePat[I];
906 FilenameBuf[J] = 0;
907
908 return FilenameBuf;
909 }
910
911 /* Returns the pointer to the environment variable
912 * string. Returns null if the env var is not set. */
getFilenamePatFromEnv(void)913 static const char *getFilenamePatFromEnv(void) {
914 const char *Filename = getenv("LLVM_PROFILE_FILE");
915 if (!Filename || !Filename[0])
916 return 0;
917 return Filename;
918 }
919
920 COMPILER_RT_VISIBILITY
__llvm_profile_get_path_prefix(void)921 const char *__llvm_profile_get_path_prefix(void) {
922 int Length;
923 char *FilenameBuf, *Prefix;
924 const char *Filename, *PrefixEnd;
925
926 if (lprofCurFilename.ProfilePathPrefix)
927 return lprofCurFilename.ProfilePathPrefix;
928
929 Length = getCurFilenameLength();
930 FilenameBuf = (char *)COMPILER_RT_ALLOCA(Length + 1);
931 Filename = getCurFilename(FilenameBuf, 0);
932 if (!Filename)
933 return "\0";
934
935 PrefixEnd = lprofFindLastDirSeparator(Filename);
936 if (!PrefixEnd)
937 return "\0";
938
939 Length = PrefixEnd - Filename + 1;
940 Prefix = (char *)malloc(Length + 1);
941 if (!Prefix) {
942 PROF_ERR("Failed to %s\n", "allocate memory.");
943 return "\0";
944 }
945 memcpy(Prefix, Filename, Length);
946 Prefix[Length] = '\0';
947 lprofCurFilename.ProfilePathPrefix = Prefix;
948 return Prefix;
949 }
950
951 COMPILER_RT_VISIBILITY
__llvm_profile_get_filename(void)952 const char *__llvm_profile_get_filename(void) {
953 int Length;
954 char *FilenameBuf;
955 const char *Filename;
956
957 Length = getCurFilenameLength();
958 FilenameBuf = (char *)malloc(Length + 1);
959 if (!FilenameBuf) {
960 PROF_ERR("Failed to %s\n", "allocate memory.");
961 return "\0";
962 }
963 Filename = getCurFilename(FilenameBuf, 1);
964 if (!Filename)
965 return "\0";
966
967 return FilenameBuf;
968 }
969
970 /* This API initializes the file handling, both user specified
971 * profile path via -fprofile-instr-generate= and LLVM_PROFILE_FILE
972 * environment variable can override this default value.
973 */
974 COMPILER_RT_VISIBILITY
__llvm_profile_initialize_file(void)975 void __llvm_profile_initialize_file(void) {
976 const char *EnvFilenamePat;
977 const char *SelectedPat = NULL;
978 ProfileNameSpecifier PNS = PNS_unknown;
979 int hasCommandLineOverrider = (INSTR_PROF_PROFILE_NAME_VAR[0] != 0);
980
981 if (__llvm_profile_counter_bias != -1)
982 lprofSetRuntimeCounterRelocation(1);
983
984 EnvFilenamePat = getFilenamePatFromEnv();
985 if (EnvFilenamePat) {
986 /* Pass CopyFilenamePat = 1, to ensure that the filename would be valid
987 at the moment when __llvm_profile_write_file() gets executed. */
988 parseAndSetFilename(EnvFilenamePat, PNS_environment, 1);
989 return;
990 } else if (hasCommandLineOverrider) {
991 SelectedPat = INSTR_PROF_PROFILE_NAME_VAR;
992 PNS = PNS_command_line;
993 } else {
994 SelectedPat = NULL;
995 PNS = PNS_default;
996 }
997
998 parseAndSetFilename(SelectedPat, PNS, 0);
999 }
1000
1001 /* This method is invoked by the runtime initialization hook
1002 * InstrProfilingRuntime.o if it is linked in.
1003 */
1004 COMPILER_RT_VISIBILITY
__llvm_profile_initialize(void)1005 void __llvm_profile_initialize(void) {
1006 __llvm_profile_initialize_file();
1007 if (!__llvm_profile_is_continuous_mode_enabled())
1008 __llvm_profile_register_write_file_atexit();
1009 }
1010
1011 /* This API is directly called by the user application code. It has the
1012 * highest precedence compared with LLVM_PROFILE_FILE environment variable
1013 * and command line option -fprofile-instr-generate=<profile_name>.
1014 */
1015 COMPILER_RT_VISIBILITY
__llvm_profile_set_filename(const char * FilenamePat)1016 void __llvm_profile_set_filename(const char *FilenamePat) {
1017 if (__llvm_profile_is_continuous_mode_enabled())
1018 return;
1019 parseAndSetFilename(FilenamePat, PNS_runtime_api, 1);
1020 }
1021
1022 /* The public API for writing profile data into the file with name
1023 * set by previous calls to __llvm_profile_set_filename or
1024 * __llvm_profile_override_default_filename or
1025 * __llvm_profile_initialize_file. */
1026 COMPILER_RT_VISIBILITY
__llvm_profile_write_file(void)1027 int __llvm_profile_write_file(void) {
1028 int rc, Length;
1029 const char *Filename;
1030 char *FilenameBuf;
1031 int PDeathSig = 0;
1032
1033 if (lprofProfileDumped() || __llvm_profile_is_continuous_mode_enabled()) {
1034 PROF_NOTE("Profile data not written to file: %s.\n", "already written");
1035 return 0;
1036 }
1037
1038 Length = getCurFilenameLength();
1039 FilenameBuf = (char *)COMPILER_RT_ALLOCA(Length + 1);
1040 Filename = getCurFilename(FilenameBuf, 0);
1041
1042 /* Check the filename. */
1043 if (!Filename) {
1044 PROF_ERR("Failed to write file : %s\n", "Filename not set");
1045 return -1;
1046 }
1047
1048 /* Check if there is llvm/runtime version mismatch. */
1049 if (GET_VERSION(__llvm_profile_get_version()) != INSTR_PROF_RAW_VERSION) {
1050 PROF_ERR("Runtime and instrumentation version mismatch : "
1051 "expected %d, but get %d\n",
1052 INSTR_PROF_RAW_VERSION,
1053 (int)GET_VERSION(__llvm_profile_get_version()));
1054 return -1;
1055 }
1056
1057 // Temporarily suspend getting SIGKILL when the parent exits.
1058 PDeathSig = lprofSuspendSigKill();
1059
1060 /* Write profile data to the file. */
1061 rc = writeFile(Filename);
1062 if (rc)
1063 PROF_ERR("Failed to write file \"%s\": %s\n", Filename, strerror(errno));
1064
1065 // Restore SIGKILL.
1066 if (PDeathSig == 1)
1067 lprofRestoreSigKill();
1068
1069 return rc;
1070 }
1071
1072 COMPILER_RT_VISIBILITY
__llvm_profile_dump(void)1073 int __llvm_profile_dump(void) {
1074 if (!doMerging())
1075 PROF_WARN("Later invocation of __llvm_profile_dump can lead to clobbering "
1076 " of previously dumped profile data : %s. Either use %%m "
1077 "in profile name or change profile name before dumping.\n",
1078 "online profile merging is not on");
1079 int rc = __llvm_profile_write_file();
1080 lprofSetProfileDumped(1);
1081 return rc;
1082 }
1083
1084 /* Order file data will be saved in a file with suffx .order. */
1085 static const char *OrderFileSuffix = ".order";
1086
1087 COMPILER_RT_VISIBILITY
__llvm_orderfile_write_file(void)1088 int __llvm_orderfile_write_file(void) {
1089 int rc, Length, LengthBeforeAppend, SuffixLength;
1090 const char *Filename;
1091 char *FilenameBuf;
1092 int PDeathSig = 0;
1093
1094 SuffixLength = strlen(OrderFileSuffix);
1095 Length = getCurFilenameLength() + SuffixLength;
1096 FilenameBuf = (char *)COMPILER_RT_ALLOCA(Length + 1);
1097 Filename = getCurFilename(FilenameBuf, 1);
1098
1099 /* Check the filename. */
1100 if (!Filename) {
1101 PROF_ERR("Failed to write file : %s\n", "Filename not set");
1102 return -1;
1103 }
1104
1105 /* Append order file suffix */
1106 LengthBeforeAppend = strlen(Filename);
1107 memcpy(FilenameBuf + LengthBeforeAppend, OrderFileSuffix, SuffixLength);
1108 FilenameBuf[LengthBeforeAppend + SuffixLength] = '\0';
1109
1110 /* Check if there is llvm/runtime version mismatch. */
1111 if (GET_VERSION(__llvm_profile_get_version()) != INSTR_PROF_RAW_VERSION) {
1112 PROF_ERR("Runtime and instrumentation version mismatch : "
1113 "expected %d, but get %d\n",
1114 INSTR_PROF_RAW_VERSION,
1115 (int)GET_VERSION(__llvm_profile_get_version()));
1116 return -1;
1117 }
1118
1119 // Temporarily suspend getting SIGKILL when the parent exits.
1120 PDeathSig = lprofSuspendSigKill();
1121
1122 /* Write order data to the file. */
1123 rc = writeOrderFile(Filename);
1124 if (rc)
1125 PROF_ERR("Failed to write file \"%s\": %s\n", Filename, strerror(errno));
1126
1127 // Restore SIGKILL.
1128 if (PDeathSig == 1)
1129 lprofRestoreSigKill();
1130
1131 return rc;
1132 }
1133
1134 COMPILER_RT_VISIBILITY
__llvm_orderfile_dump(void)1135 int __llvm_orderfile_dump(void) {
1136 int rc = __llvm_orderfile_write_file();
1137 return rc;
1138 }
1139
writeFileWithoutReturn(void)1140 static void writeFileWithoutReturn(void) { __llvm_profile_write_file(); }
1141
1142 COMPILER_RT_VISIBILITY
__llvm_profile_register_write_file_atexit(void)1143 int __llvm_profile_register_write_file_atexit(void) {
1144 static int HasBeenRegistered = 0;
1145
1146 if (HasBeenRegistered)
1147 return 0;
1148
1149 lprofSetupValueProfiler();
1150
1151 HasBeenRegistered = 1;
1152 return atexit(writeFileWithoutReturn);
1153 }
1154
1155 #endif
1156