1 // Protocol Buffers - Google's data interchange format
2 // Copyright 2008 Google Inc. All rights reserved.
3 //
4 // Use of this source code is governed by a BSD-style
5 // license that can be found in the LICENSE file or at
6 // https://developers.google.com/open-source/licenses/bsd
7
8 #ifndef GOOGLE_PROTOBUF_SRC_GOOGLE_PROTOBUF_ARENAZ_SAMPLER_H__
9 #define GOOGLE_PROTOBUF_SRC_GOOGLE_PROTOBUF_ARENAZ_SAMPLER_H__
10
11 #include <array>
12 #include <atomic>
13 #include <cstddef>
14 #include <cstdint>
15 #include <utility>
16
17
18 // Must be included last.
19 #include "google/protobuf/port_def.inc"
20
21 namespace google {
22 namespace protobuf {
23 namespace internal {
24
25 #if defined(PROTOBUF_ARENAZ_SAMPLE)
26 struct ThreadSafeArenaStats;
27 void RecordAllocateSlow(ThreadSafeArenaStats* info, size_t used,
28 size_t allocated, size_t wasted);
29 // Stores information about a sampled thread safe arena. All mutations to this
30 // *must* be made through `Record*` functions below. All reads from this *must*
31 // only occur in the callback to `ThreadSafeArenazSampler::Iterate`.
32 struct ThreadSafeArenaStats
33 : public absl::profiling_internal::Sample<ThreadSafeArenaStats> {
34 // Constructs the object but does not fill in any fields.
35 ThreadSafeArenaStats();
36 ~ThreadSafeArenaStats();
37
38 // Puts the object into a clean state, fills in the logically `const` members,
39 // blocking for any readers that are currently sampling the object. The
40 // 'stride' parameter is the number of ThreadSafeArenas that were instantiated
41 // between this sample and the previous one.
42 void PrepareForSampling(int64_t stride)
43 ABSL_EXCLUSIVE_LOCKS_REQUIRED(init_mu);
44
45 // These fields are mutated by the various Record* APIs and need to be
46 // thread-safe.
47 struct BlockStats {
48 std::atomic<int> num_allocations;
49 std::atomic<size_t> bytes_allocated;
50 std::atomic<size_t> bytes_used;
51 std::atomic<size_t> bytes_wasted;
52
53 void PrepareForSampling();
54 };
55
56 // block_histogram is a kBlockHistogramBins sized histogram. The zeroth bin
57 // stores info about blocks of size \in [1, 1 << kLogMaxSizeForBinZero]. Bin
58 // i, where i > 0, stores info for blocks of size \in (max_size_bin (i-1),
59 // 1 << (kLogMaxSizeForBinZero + i)]. The final bin stores info about blocks
60 // of size \in [kMaxSizeForPenultimateBin + 1,
61 // std::numeric_limits<size_t>::max()].
62 static constexpr size_t kBlockHistogramBins = 15;
63 static constexpr size_t kLogMaxSizeForBinZero = 7;
64 static constexpr size_t kMaxSizeForBinZero = (1 << kLogMaxSizeForBinZero);
65 static constexpr size_t kMaxSizeForPenultimateBin =
66 1 << (kLogMaxSizeForBinZero + kBlockHistogramBins - 2);
67 std::array<BlockStats, kBlockHistogramBins> block_histogram;
68
69 // Records the largest block allocated for the arena.
70 std::atomic<size_t> max_block_size;
71 // Bit `i` is set to 1 indicates that a thread with `tid % 63 = i` accessed
72 // the underlying arena. We use `% 63` as a rudimentary hash to ensure some
73 // bit mixing for thread-ids; `% 64` would only grab the low bits and might
74 // create sampling artifacts.
75 std::atomic<uint64_t> thread_ids;
76
77 // All of the fields below are set by `PrepareForSampling`, they must not
78 // be mutated in `Record*` functions. They are logically `const` in that
79 // sense. These are guarded by init_mu, but that is not externalized to
80 // clients, who can only read them during
81 // `ThreadSafeArenazSampler::Iterate` which will hold the lock.
82 static constexpr int kMaxStackDepth = 64;
83 int32_t depth;
84 void* stack[kMaxStackDepth];
RecordAllocateStatsThreadSafeArenaStats85 static void RecordAllocateStats(ThreadSafeArenaStats* info, size_t used,
86 size_t allocated, size_t wasted) {
87 if (PROTOBUF_PREDICT_TRUE(info == nullptr)) return;
88 RecordAllocateSlow(info, used, allocated, wasted);
89 }
90
91 // Returns the bin for the provided size.
92 static size_t FindBin(size_t bytes);
93
94 // Returns the min and max bytes that can be stored in the histogram for
95 // blocks in the provided bin.
96 static std::pair<size_t, size_t> MinMaxBlockSizeForBin(size_t bin);
97 };
98
99 struct SamplingState {
100 // Number of ThreadSafeArenas that should be instantiated before the next
101 // ThreadSafeArena is sampled. This variable is decremented with each
102 // instantiation.
103 int64_t next_sample;
104 // When we make a sampling decision, we record that distance between from the
105 // previous sample so we can weight each sample. 'distance' here is the
106 // number of instantiations of ThreadSafeArena.
107 int64_t sample_stride;
108 };
109
110 ThreadSafeArenaStats* SampleSlow(SamplingState& sampling_state);
111 void UnsampleSlow(ThreadSafeArenaStats* info);
112
113 class ThreadSafeArenaStatsHandle {
114 public:
115 explicit ThreadSafeArenaStatsHandle() = default;
ThreadSafeArenaStatsHandle(ThreadSafeArenaStats * info)116 explicit ThreadSafeArenaStatsHandle(ThreadSafeArenaStats* info)
117 : info_(info) {}
118
~ThreadSafeArenaStatsHandle()119 ~ThreadSafeArenaStatsHandle() {
120 if (PROTOBUF_PREDICT_TRUE(info_ == nullptr)) return;
121 UnsampleSlow(info_);
122 }
123
ThreadSafeArenaStatsHandle(ThreadSafeArenaStatsHandle && other)124 ThreadSafeArenaStatsHandle(ThreadSafeArenaStatsHandle&& other) noexcept
125 : info_(std::exchange(other.info_, nullptr)) {}
126
127 ThreadSafeArenaStatsHandle& operator=(
128 ThreadSafeArenaStatsHandle&& other) noexcept {
129 if (PROTOBUF_PREDICT_FALSE(info_ != nullptr)) {
130 UnsampleSlow(info_);
131 }
132 info_ = std::exchange(other.info_, nullptr);
133 return *this;
134 }
135
MutableStats()136 ThreadSafeArenaStats* MutableStats() { return info_; }
137
swap(ThreadSafeArenaStatsHandle & lhs,ThreadSafeArenaStatsHandle & rhs)138 friend void swap(ThreadSafeArenaStatsHandle& lhs,
139 ThreadSafeArenaStatsHandle& rhs) {
140 std::swap(lhs.info_, rhs.info_);
141 }
142
143 friend class ThreadSafeArenaStatsHandlePeer;
144
145 private:
146 ThreadSafeArenaStats* info_ = nullptr;
147 };
148
149 using ThreadSafeArenazSampler =
150 ::absl::profiling_internal::SampleRecorder<ThreadSafeArenaStats>;
151
152 extern PROTOBUF_THREAD_LOCAL SamplingState global_sampling_state;
153
154 // Returns an RAII sampling handle that manages registration and unregistation
155 // with the global sampler.
Sample()156 inline ThreadSafeArenaStatsHandle Sample() {
157 if (PROTOBUF_PREDICT_TRUE(--global_sampling_state.next_sample > 0)) {
158 return ThreadSafeArenaStatsHandle(nullptr);
159 }
160 return ThreadSafeArenaStatsHandle(SampleSlow(global_sampling_state));
161 }
162
163 #else
164
165 using SamplingState = int64_t;
166
167 struct ThreadSafeArenaStats {
168 static void RecordAllocateStats(ThreadSafeArenaStats*, size_t /*requested*/,
169 size_t /*allocated*/, size_t /*wasted*/) {}
170 };
171
172 ThreadSafeArenaStats* SampleSlow(SamplingState& next_sample);
173 void UnsampleSlow(ThreadSafeArenaStats* info);
174
175 class ThreadSafeArenaStatsHandle {
176 public:
177 explicit ThreadSafeArenaStatsHandle() = default;
178 explicit ThreadSafeArenaStatsHandle(ThreadSafeArenaStats*) {}
179
180 void RecordReset() {}
181
182 ThreadSafeArenaStats* MutableStats() { return nullptr; }
183
184 friend void swap(ThreadSafeArenaStatsHandle&, ThreadSafeArenaStatsHandle&) {}
185
186 private:
187 friend class ThreadSafeArenaStatsHandlePeer;
188 };
189
190 class ThreadSafeArenazSampler {
191 public:
192 void Unregister(ThreadSafeArenaStats*) {}
193 void SetMaxSamples(int32_t) {}
194 };
195
196 // Returns an RAII sampling handle that manages registration and unregistation
197 // with the global sampler.
198 inline ThreadSafeArenaStatsHandle Sample() {
199 return ThreadSafeArenaStatsHandle(nullptr);
200 }
201 #endif // defined(PROTOBUF_ARENAZ_SAMPLE)
202
203 // Returns a global Sampler.
204 ThreadSafeArenazSampler& GlobalThreadSafeArenazSampler();
205
206 using ThreadSafeArenazConfigListener = void (*)();
207 void SetThreadSafeArenazConfigListener(ThreadSafeArenazConfigListener l);
208
209 // Enables or disables sampling for thread safe arenas.
210 void SetThreadSafeArenazEnabled(bool enabled);
211 void SetThreadSafeArenazEnabledInternal(bool enabled);
212
213 // Returns true if sampling is on, false otherwise.
214 bool IsThreadSafeArenazEnabled();
215
216 // Sets the rate at which thread safe arena will be sampled.
217 void SetThreadSafeArenazSampleParameter(int32_t rate);
218 void SetThreadSafeArenazSampleParameterInternal(int32_t rate);
219
220 // Returns the rate at which thread safe arena will be sampled.
221 int32_t ThreadSafeArenazSampleParameter();
222
223 // Sets a soft max for the number of samples that will be kept.
224 void SetThreadSafeArenazMaxSamples(int32_t max);
225 void SetThreadSafeArenazMaxSamplesInternal(int32_t max);
226
227 // Returns the max number of samples that will be kept.
228 size_t ThreadSafeArenazMaxSamples();
229
230 // Sets the current value for when arenas should be next sampled.
231 void SetThreadSafeArenazGlobalNextSample(int64_t next_sample);
232
233 } // namespace internal
234 } // namespace protobuf
235 } // namespace google
236
237 #include "google/protobuf/port_undef.inc"
238 #endif // GOOGLE_PROTOBUF_SRC_PROTOBUF_ARENAZ_SAMPLER_H__
239