1 /*===- InstrProfilingValue.c - Support library for PGO instrumentation ----===*\
2 |*
3 |* The LLVM Compiler Infrastructure
4 |*
5 |* This file is distributed under the University of Illinois Open Source
6 |* License. See LICENSE.TXT for details.
7 |*
8 \*===----------------------------------------------------------------------===*/
9
10 #include "InstrProfiling.h"
11 #include "InstrProfilingInternal.h"
12 #include <limits.h>
13 #include <stdio.h>
14 #include <stdlib.h>
15 #include <string.h>
16 #define INSTR_PROF_VALUE_PROF_DATA
17 #define INSTR_PROF_COMMON_API_IMPL
18 #include "InstrProfData.inc"
19
20 #define PROF_OOM(Msg) PROF_ERR(Msg ":%s\n", "Out of memory");
21 #define PROF_OOM_RETURN(Msg) \
22 { \
23 PROF_OOM(Msg) \
24 return 0; \
25 }
26
27 #if COMPILER_RT_HAS_ATOMICS != 1
28 COMPILER_RT_VISIBILITY
BoolCmpXchg(void ** Ptr,void * OldV,void * NewV)29 uint32_t BoolCmpXchg(void **Ptr, void *OldV, void *NewV) {
30 void *R = *Ptr;
31 if (R == OldV) {
32 *Ptr = NewV;
33 return 1;
34 }
35 return 0;
36 }
37 #endif
38
39 /* This method is only used in value profiler mock testing. */
40 COMPILER_RT_VISIBILITY void
__llvm_profile_set_num_value_sites(__llvm_profile_data * Data,uint32_t ValueKind,uint16_t NumValueSites)41 __llvm_profile_set_num_value_sites(__llvm_profile_data *Data,
42 uint32_t ValueKind, uint16_t NumValueSites) {
43 *((uint16_t *)&Data->NumValueSites[ValueKind]) = NumValueSites;
44 }
45
46 /* This method is only used in value profiler mock testing. */
47 COMPILER_RT_VISIBILITY const __llvm_profile_data *
__llvm_profile_iterate_data(const __llvm_profile_data * Data)48 __llvm_profile_iterate_data(const __llvm_profile_data *Data) {
49 return Data + 1;
50 }
51
52 /* This method is only used in value profiler mock testing. */
53 COMPILER_RT_VISIBILITY void *
__llvm_get_function_addr(const __llvm_profile_data * Data)54 __llvm_get_function_addr(const __llvm_profile_data *Data) {
55 return Data->FunctionPointer;
56 }
57
58 /* Allocate an array that holds the pointers to the linked lists of
59 * value profile counter nodes. The number of element of the array
60 * is the total number of value profile sites instrumented. Returns
61 * 0 if allocation fails.
62 */
63
allocateValueProfileCounters(__llvm_profile_data * Data)64 static int allocateValueProfileCounters(__llvm_profile_data *Data) {
65 uint64_t NumVSites = 0;
66 uint32_t VKI;
67 for (VKI = IPVK_First; VKI <= IPVK_Last; ++VKI)
68 NumVSites += Data->NumValueSites[VKI];
69
70 ValueProfNode **Mem =
71 (ValueProfNode **)calloc(NumVSites, sizeof(ValueProfNode *));
72 if (!Mem)
73 return 0;
74 if (!COMPILER_RT_BOOL_CMPXCHG(&Data->Values, 0, Mem)) {
75 free(Mem);
76 return 0;
77 }
78 return 1;
79 }
80
deallocateValueProfileCounters(__llvm_profile_data * Data)81 static void deallocateValueProfileCounters(__llvm_profile_data *Data) {
82 uint64_t NumVSites = 0, I;
83 uint32_t VKI;
84 if (!Data->Values)
85 return;
86 for (VKI = IPVK_First; VKI <= IPVK_Last; ++VKI)
87 NumVSites += Data->NumValueSites[VKI];
88 for (I = 0; I < NumVSites; I++) {
89 ValueProfNode *Node = ((ValueProfNode **)Data->Values)[I];
90 while (Node) {
91 ValueProfNode *Next = Node->Next;
92 free(Node);
93 Node = Next;
94 }
95 }
96 free(Data->Values);
97 }
98
99 COMPILER_RT_VISIBILITY void
__llvm_profile_instrument_target(uint64_t TargetValue,void * Data,uint32_t CounterIndex)100 __llvm_profile_instrument_target(uint64_t TargetValue, void *Data,
101 uint32_t CounterIndex) {
102
103 __llvm_profile_data *PData = (__llvm_profile_data *)Data;
104 if (!PData)
105 return;
106
107 if (!PData->Values) {
108 if (!allocateValueProfileCounters(PData))
109 return;
110 }
111
112 ValueProfNode **ValueCounters = (ValueProfNode **)PData->Values;
113 ValueProfNode *PrevVNode = NULL;
114 ValueProfNode *CurrentVNode = ValueCounters[CounterIndex];
115
116 uint8_t VDataCount = 0;
117 while (CurrentVNode) {
118 if (TargetValue == CurrentVNode->VData.Value) {
119 CurrentVNode->VData.Count++;
120 return;
121 }
122 PrevVNode = CurrentVNode;
123 CurrentVNode = CurrentVNode->Next;
124 ++VDataCount;
125 }
126
127 if (VDataCount >= UCHAR_MAX)
128 return;
129
130 CurrentVNode = (ValueProfNode *)calloc(1, sizeof(ValueProfNode));
131 if (!CurrentVNode)
132 return;
133
134 CurrentVNode->VData.Value = TargetValue;
135 CurrentVNode->VData.Count++;
136
137 uint32_t Success = 0;
138 if (!ValueCounters[CounterIndex])
139 Success =
140 COMPILER_RT_BOOL_CMPXCHG(&ValueCounters[CounterIndex], 0, CurrentVNode);
141 else if (PrevVNode && !PrevVNode->Next)
142 Success = COMPILER_RT_BOOL_CMPXCHG(&(PrevVNode->Next), 0, CurrentVNode);
143
144 if (!Success) {
145 free(CurrentVNode);
146 return;
147 }
148 }
149
150 /* For multi-threaded programs, while the profile is being dumped, other
151 threads may still be updating the value profile data and creating new
152 value entries. To accommadate this, we need to add extra bytes to the
153 data buffer. The size of the extra space is controlled by an environment
154 variable. */
getVprofExtraBytes()155 static unsigned getVprofExtraBytes() {
156 const char *ExtraStr =
157 GetEnvHook ? GetEnvHook("LLVM_VALUE_PROF_BUFFER_EXTRA") : 0;
158 if (!ExtraStr || !ExtraStr[0])
159 return 1024;
160 return (unsigned)atoi(ExtraStr);
161 }
162
163 /* Extract the value profile data info from the runtime. */
164 #define DEF_VALUE_RECORD(R, NS, V) \
165 ValueProfRuntimeRecord R; \
166 if (initializeValueProfRuntimeRecord(&R, NS, V)) \
167 PROF_OOM_RETURN("Failed to write value profile data ");
168
169 #define DTOR_VALUE_RECORD(R) finalizeValueProfRuntimeRecord(&R);
170
171 COMPILER_RT_VISIBILITY uint64_t
__llvm_profile_gather_value_data(uint8_t ** VDataArray)172 __llvm_profile_gather_value_data(uint8_t **VDataArray) {
173 size_t S = 0, RealSize = 0, BufferCapacity = 0, Extra = 0;
174 __llvm_profile_data *I;
175 if (!VDataArray)
176 PROF_OOM_RETURN("Failed to write value profile data ");
177
178 const __llvm_profile_data *DataEnd = __llvm_profile_end_data();
179 const __llvm_profile_data *DataBegin = __llvm_profile_begin_data();
180
181 /*
182 * Compute the total Size of the buffer to hold ValueProfData
183 * structures for functions with value profile data.
184 */
185 for (I = (__llvm_profile_data *)DataBegin; I != DataEnd; ++I) {
186
187 DEF_VALUE_RECORD(R, I->NumValueSites, I->Values);
188
189 /* Compute the size of ValueProfData from this runtime record. */
190 if (getNumValueKindsRT(&R) != 0)
191 S += getValueProfDataSizeRT(&R);
192
193 DTOR_VALUE_RECORD(R);
194 }
195 /* No value sites or no value profile data is collected. */
196 if (!S)
197 return 0;
198
199 Extra = getVprofExtraBytes();
200 BufferCapacity = S + Extra;
201 *VDataArray = calloc(BufferCapacity, sizeof(uint8_t));
202 if (!*VDataArray)
203 PROF_OOM_RETURN("Failed to write value profile data ");
204
205 ValueProfData *VD = (ValueProfData *)(*VDataArray);
206 /*
207 * Extract value profile data and write into ValueProfData structure
208 * one by one. Note that new value profile data added to any value
209 * site (from another thread) after the ValueProfRuntimeRecord is
210 * initialized (when the profile data snapshot is taken) won't be
211 * collected. This is not a problem as those dropped value will have
212 * very low taken count.
213 */
214 for (I = (__llvm_profile_data *)DataBegin; I != DataEnd; ++I) {
215 DEF_VALUE_RECORD(R, I->NumValueSites, I->Values);
216 if (getNumValueKindsRT(&R) == 0)
217 continue;
218
219 /* Record R has taken a snapshot of the VP data at this point. Newly
220 added VP data for this function will be dropped. */
221 /* Check if there is enough space. */
222 if (BufferCapacity - RealSize < getValueProfDataSizeRT(&R)) {
223 PROF_ERR("Value profile data is dropped :%s \n",
224 "Out of buffer space. Use environment "
225 " LLVM_VALUE_PROF_BUFFER_EXTRA to allocate more");
226 I->Values = 0;
227 }
228
229 serializeValueProfDataFromRT(&R, VD);
230 deallocateValueProfileCounters(I);
231 I->Values = VD;
232 RealSize += VD->TotalSize;
233 VD = (ValueProfData *)((char *)VD + VD->TotalSize);
234 DTOR_VALUE_RECORD(R);
235 }
236
237 return RealSize;
238 }
239