1 /*
2 * Copyright (C) 2018 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "perfetto/ext/base/watchdog.h"
18
19 #if PERFETTO_BUILDFLAG(PERFETTO_WATCHDOG)
20
21 #include <fcntl.h>
22 #include <inttypes.h>
23 #include <signal.h>
24 #include <stdint.h>
25
26 #include <fstream>
27 #include <thread>
28
29 #include "perfetto/base/build_config.h"
30 #include "perfetto/base/logging.h"
31 #include "perfetto/base/thread_utils.h"
32 #include "perfetto/ext/base/scoped_file.h"
33
34 namespace perfetto {
35 namespace base {
36
37 namespace {
38
39 constexpr uint32_t kDefaultPollingInterval = 30 * 1000;
40
IsMultipleOf(uint32_t number,uint32_t divisor)41 bool IsMultipleOf(uint32_t number, uint32_t divisor) {
42 return number >= divisor && number % divisor == 0;
43 }
44
MeanForArray(const uint64_t array[],size_t size)45 double MeanForArray(const uint64_t array[], size_t size) {
46 uint64_t total = 0;
47 for (size_t i = 0; i < size; i++) {
48 total += array[i];
49 }
50 return static_cast<double>(total / size);
51
52 }
53
54 } // namespace
55
ReadProcStat(int fd,ProcStat * out)56 bool ReadProcStat(int fd, ProcStat* out) {
57 char c[512];
58 size_t c_pos = 0;
59 while (c_pos < sizeof(c) - 1) {
60 ssize_t rd = PERFETTO_EINTR(read(fd, c + c_pos, sizeof(c) - c_pos));
61 if (rd < 0) {
62 PERFETTO_ELOG("Failed to read stat file to enforce resource limits.");
63 return false;
64 }
65 if (rd == 0)
66 break;
67 c_pos += static_cast<size_t>(rd);
68 }
69 PERFETTO_CHECK(c_pos < sizeof(c));
70 c[c_pos] = '\0';
71
72 if (sscanf(c,
73 "%*d %*s %*c %*d %*d %*d %*d %*d %*u %*u %*u %*u %*u %lu"
74 "%lu %*d %*d %*d %*d %*d %*d %*u %*u %ld",
75 &out->utime, &out->stime, &out->rss_pages) != 3) {
76 PERFETTO_ELOG("Invalid stat format: %s", c);
77 return false;
78 }
79 return true;
80 }
81
Watchdog(uint32_t polling_interval_ms)82 Watchdog::Watchdog(uint32_t polling_interval_ms)
83 : polling_interval_ms_(polling_interval_ms) {}
84
~Watchdog()85 Watchdog::~Watchdog() {
86 if (!thread_.joinable()) {
87 PERFETTO_DCHECK(!enabled_);
88 return;
89 }
90 PERFETTO_DCHECK(enabled_);
91 enabled_ = false;
92 exit_signal_.notify_one();
93 thread_.join();
94 }
95
GetInstance()96 Watchdog* Watchdog::GetInstance() {
97 static Watchdog* watchdog = new Watchdog(kDefaultPollingInterval);
98 return watchdog;
99 }
100
CreateFatalTimer(uint32_t ms)101 Watchdog::Timer Watchdog::CreateFatalTimer(uint32_t ms) {
102 if (!enabled_.load(std::memory_order_relaxed))
103 return Watchdog::Timer(0);
104
105 return Watchdog::Timer(ms);
106 }
107
Start()108 void Watchdog::Start() {
109 std::lock_guard<std::mutex> guard(mutex_);
110 if (thread_.joinable()) {
111 PERFETTO_DCHECK(enabled_);
112 } else {
113 PERFETTO_DCHECK(!enabled_);
114
115 #if PERFETTO_BUILDFLAG(PERFETTO_OS_LINUX) || \
116 PERFETTO_BUILDFLAG(PERFETTO_OS_ANDROID)
117 // Kick the thread to start running but only on Android or Linux.
118 enabled_ = true;
119 thread_ = std::thread(&Watchdog::ThreadMain, this);
120 #endif
121 }
122 }
123
SetMemoryLimit(uint64_t bytes,uint32_t window_ms)124 void Watchdog::SetMemoryLimit(uint64_t bytes, uint32_t window_ms) {
125 // Update the fields under the lock.
126 std::lock_guard<std::mutex> guard(mutex_);
127
128 PERFETTO_CHECK(IsMultipleOf(window_ms, polling_interval_ms_) || bytes == 0);
129
130 size_t size = bytes == 0 ? 0 : window_ms / polling_interval_ms_ + 1;
131 memory_window_bytes_.Reset(size);
132 memory_limit_bytes_ = bytes;
133 }
134
SetCpuLimit(uint32_t percentage,uint32_t window_ms)135 void Watchdog::SetCpuLimit(uint32_t percentage, uint32_t window_ms) {
136 std::lock_guard<std::mutex> guard(mutex_);
137
138 PERFETTO_CHECK(percentage <= 100);
139 PERFETTO_CHECK(IsMultipleOf(window_ms, polling_interval_ms_) ||
140 percentage == 0);
141
142 size_t size = percentage == 0 ? 0 : window_ms / polling_interval_ms_ + 1;
143 cpu_window_time_ticks_.Reset(size);
144 cpu_limit_percentage_ = percentage;
145 }
146
ThreadMain()147 void Watchdog::ThreadMain() {
148 base::ScopedFile stat_fd(base::OpenFile("/proc/self/stat", O_RDONLY));
149 if (!stat_fd) {
150 PERFETTO_ELOG("Failed to open stat file to enforce resource limits.");
151 return;
152 }
153
154 std::unique_lock<std::mutex> guard(mutex_);
155 for (;;) {
156 exit_signal_.wait_for(guard,
157 std::chrono::milliseconds(polling_interval_ms_));
158 if (!enabled_)
159 return;
160
161 lseek(stat_fd.get(), 0, SEEK_SET);
162
163 ProcStat stat;
164 if (!ReadProcStat(stat_fd.get(), &stat)) {
165 return;
166 }
167
168 uint64_t cpu_time = stat.utime + stat.stime;
169 uint64_t rss_bytes =
170 static_cast<uint64_t>(stat.rss_pages) * base::kPageSize;
171
172 CheckMemory(rss_bytes);
173 CheckCpu(cpu_time);
174 }
175 }
176
CheckMemory(uint64_t rss_bytes)177 void Watchdog::CheckMemory(uint64_t rss_bytes) {
178 if (memory_limit_bytes_ == 0)
179 return;
180
181 // Add the current stat value to the ring buffer and check that the mean
182 // remains under our threshold.
183 if (memory_window_bytes_.Push(rss_bytes)) {
184 if (memory_window_bytes_.Mean() > static_cast<double>(memory_limit_bytes_)) {
185 PERFETTO_ELOG(
186 "Memory watchdog trigger. Memory window of %f bytes is above the "
187 "%" PRIu64 " bytes limit.",
188 memory_window_bytes_.Mean(), memory_limit_bytes_);
189 kill(getpid(), SIGABRT);
190 }
191 }
192 }
193
CheckCpu(uint64_t cpu_time)194 void Watchdog::CheckCpu(uint64_t cpu_time) {
195 if (cpu_limit_percentage_ == 0)
196 return;
197
198 // Add the cpu time to the ring buffer.
199 if (cpu_window_time_ticks_.Push(cpu_time)) {
200 // Compute the percentage over the whole window and check that it remains
201 // under the threshold.
202 uint64_t difference_ticks = cpu_window_time_ticks_.NewestWhenFull() -
203 cpu_window_time_ticks_.OldestWhenFull();
204 double window_interval_ticks =
205 (static_cast<double>(WindowTimeForRingBuffer(cpu_window_time_ticks_)) /
206 1000.0) *
207 static_cast<double>(sysconf(_SC_CLK_TCK));
208 double percentage = static_cast<double>(difference_ticks) /
209 static_cast<double>(window_interval_ticks) * 100;
210 if (percentage > cpu_limit_percentage_) {
211 PERFETTO_ELOG("CPU watchdog trigger. %f%% CPU use is above the %" PRIu32
212 "%% CPU limit.",
213 percentage, cpu_limit_percentage_);
214 kill(getpid(), SIGABRT);
215 }
216 }
217 }
218
WindowTimeForRingBuffer(const WindowedInterval & window)219 uint32_t Watchdog::WindowTimeForRingBuffer(const WindowedInterval& window) {
220 return static_cast<uint32_t>(window.size() - 1) * polling_interval_ms_;
221 }
222
Push(uint64_t sample)223 bool Watchdog::WindowedInterval::Push(uint64_t sample) {
224 // Add the sample to the current position in the ring buffer.
225 buffer_[position_] = sample;
226
227 // Update the position with next one circularily.
228 position_ = (position_ + 1) % size_;
229
230 // Set the filled flag the first time we wrap.
231 filled_ = filled_ || position_ == 0;
232 return filled_;
233 }
234
Mean() const235 double Watchdog::WindowedInterval::Mean() const {
236 return MeanForArray(buffer_.get(), size_);
237 }
238
Clear()239 void Watchdog::WindowedInterval::Clear() {
240 position_ = 0;
241 buffer_.reset(new uint64_t[size_]());
242 }
243
Reset(size_t new_size)244 void Watchdog::WindowedInterval::Reset(size_t new_size) {
245 position_ = 0;
246 size_ = new_size;
247 buffer_.reset(new_size == 0 ? nullptr : new uint64_t[new_size]());
248 }
249
Timer(uint32_t ms)250 Watchdog::Timer::Timer(uint32_t ms) {
251 if (!ms)
252 return; // No-op timer created when the watchdog is disabled.
253
254 struct sigevent sev = {};
255 sev.sigev_notify = SIGEV_THREAD_ID;
256 sev._sigev_un._tid = base::GetThreadId();
257 sev.sigev_signo = SIGABRT;
258 PERFETTO_CHECK(timer_create(CLOCK_MONOTONIC, &sev, &timerid_) != -1);
259 struct itimerspec its = {};
260 its.it_value.tv_sec = ms / 1000;
261 its.it_value.tv_nsec = 1000000L * (ms % 1000);
262 PERFETTO_CHECK(timer_settime(timerid_, 0, &its, nullptr) != -1);
263 }
264
~Timer()265 Watchdog::Timer::~Timer() {
266 if (timerid_ != nullptr) {
267 timer_delete(timerid_);
268 }
269 }
270
Timer(Timer && other)271 Watchdog::Timer::Timer(Timer&& other) noexcept {
272 timerid_ = other.timerid_;
273 other.timerid_ = nullptr;
274 }
275
276 } // namespace base
277 } // namespace perfetto
278
279 #endif // PERFETTO_BUILDFLAG(PERFETTO_WATCHDOG)
280