1 /*
2 * Copyright 2019 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 //#define LOG_NDEBUG 0
18 #define ATRACE_TAG ATRACE_TAG_GRAPHICS
19 #undef LOG_TAG
20 #define LOG_TAG "RegionSamplingThread"
21
22 #include "RegionSamplingThread.h"
23
24 #include <cutils/properties.h>
25 #include <gui/IRegionSamplingListener.h>
26 #include <utils/Trace.h>
27 #include <string>
28
29 #include <compositionengine/Display.h>
30 #include <compositionengine/impl/OutputCompositionState.h>
31 #include "DisplayDevice.h"
32 #include "Layer.h"
33 #include "SurfaceFlinger.h"
34
35 namespace android {
36 using namespace std::chrono_literals;
37
38 template <typename T>
39 struct SpHash {
operator ()android::SpHash40 size_t operator()(const sp<T>& p) const { return std::hash<T*>()(p.get()); }
41 };
42
43 constexpr auto lumaSamplingStepTag = "LumaSamplingStep";
44 enum class samplingStep {
45 noWorkNeeded,
46 idleTimerWaiting,
47 waitForZeroPhase,
48 waitForSamplePhase,
49 sample
50 };
51
52 constexpr auto defaultRegionSamplingOffset = -3ms;
53 constexpr auto defaultRegionSamplingPeriod = 100ms;
54 constexpr auto defaultRegionSamplingTimerTimeout = 100ms;
55 // TODO: (b/127403193) duration to string conversion could probably be constexpr
56 template <typename Rep, typename Per>
toNsString(std::chrono::duration<Rep,Per> t)57 inline std::string toNsString(std::chrono::duration<Rep, Per> t) {
58 return std::to_string(std::chrono::duration_cast<std::chrono::nanoseconds>(t).count());
59 }
60
EnvironmentTimingTunables()61 RegionSamplingThread::EnvironmentTimingTunables::EnvironmentTimingTunables() {
62 char value[PROPERTY_VALUE_MAX] = {};
63
64 property_get("debug.sf.region_sampling_offset_ns", value,
65 toNsString(defaultRegionSamplingOffset).c_str());
66 int const samplingOffsetNsRaw = atoi(value);
67
68 property_get("debug.sf.region_sampling_period_ns", value,
69 toNsString(defaultRegionSamplingPeriod).c_str());
70 int const samplingPeriodNsRaw = atoi(value);
71
72 property_get("debug.sf.region_sampling_timer_timeout_ns", value,
73 toNsString(defaultRegionSamplingTimerTimeout).c_str());
74 int const samplingTimerTimeoutNsRaw = atoi(value);
75
76 if ((samplingPeriodNsRaw < 0) || (samplingTimerTimeoutNsRaw < 0)) {
77 ALOGW("User-specified sampling tuning options nonsensical. Using defaults");
78 mSamplingOffset = defaultRegionSamplingOffset;
79 mSamplingPeriod = defaultRegionSamplingPeriod;
80 mSamplingTimerTimeout = defaultRegionSamplingTimerTimeout;
81 } else {
82 mSamplingOffset = std::chrono::nanoseconds(samplingOffsetNsRaw);
83 mSamplingPeriod = std::chrono::nanoseconds(samplingPeriodNsRaw);
84 mSamplingTimerTimeout = std::chrono::nanoseconds(samplingTimerTimeoutNsRaw);
85 }
86 }
87
88 struct SamplingOffsetCallback : DispSync::Callback {
SamplingOffsetCallbackandroid::SamplingOffsetCallback89 SamplingOffsetCallback(RegionSamplingThread& samplingThread, Scheduler& scheduler,
90 std::chrono::nanoseconds targetSamplingOffset)
91 : mRegionSamplingThread(samplingThread),
92 mScheduler(scheduler),
93 mTargetSamplingOffset(targetSamplingOffset) {}
94
~SamplingOffsetCallbackandroid::SamplingOffsetCallback95 ~SamplingOffsetCallback() { stopVsyncListener(); }
96
97 SamplingOffsetCallback(const SamplingOffsetCallback&) = delete;
98 SamplingOffsetCallback& operator=(const SamplingOffsetCallback&) = delete;
99
startVsyncListenerandroid::SamplingOffsetCallback100 void startVsyncListener() {
101 std::lock_guard lock(mMutex);
102 if (mVsyncListening) return;
103
104 mPhaseIntervalSetting = Phase::ZERO;
105 mScheduler.withPrimaryDispSync([this](android::DispSync& sync) {
106 sync.addEventListener("SamplingThreadDispSyncListener", 0, this, mLastCallbackTime);
107 });
108 mVsyncListening = true;
109 }
110
stopVsyncListenerandroid::SamplingOffsetCallback111 void stopVsyncListener() {
112 std::lock_guard lock(mMutex);
113 stopVsyncListenerLocked();
114 }
115
116 private:
stopVsyncListenerLockedandroid::SamplingOffsetCallback117 void stopVsyncListenerLocked() /*REQUIRES(mMutex)*/ {
118 if (!mVsyncListening) return;
119
120 mScheduler.withPrimaryDispSync([this](android::DispSync& sync) {
121 sync.removeEventListener(this, &mLastCallbackTime);
122 });
123 mVsyncListening = false;
124 }
125
onDispSyncEventandroid::SamplingOffsetCallback126 void onDispSyncEvent(nsecs_t /* when */) final {
127 std::unique_lock<decltype(mMutex)> lock(mMutex);
128
129 if (mPhaseIntervalSetting == Phase::ZERO) {
130 ATRACE_INT(lumaSamplingStepTag, static_cast<int>(samplingStep::waitForSamplePhase));
131 mPhaseIntervalSetting = Phase::SAMPLING;
132 mScheduler.withPrimaryDispSync([this](android::DispSync& sync) {
133 sync.changePhaseOffset(this, mTargetSamplingOffset.count());
134 });
135 return;
136 }
137
138 if (mPhaseIntervalSetting == Phase::SAMPLING) {
139 mPhaseIntervalSetting = Phase::ZERO;
140 mScheduler.withPrimaryDispSync(
141 [this](android::DispSync& sync) { sync.changePhaseOffset(this, 0); });
142 stopVsyncListenerLocked();
143 lock.unlock();
144 mRegionSamplingThread.notifySamplingOffset();
145 return;
146 }
147 }
148
149 RegionSamplingThread& mRegionSamplingThread;
150 Scheduler& mScheduler;
151 const std::chrono::nanoseconds mTargetSamplingOffset;
152 mutable std::mutex mMutex;
153 nsecs_t mLastCallbackTime = 0;
154 enum class Phase {
155 ZERO,
156 SAMPLING
157 } mPhaseIntervalSetting /*GUARDED_BY(mMutex) macro doesnt work with unique_lock?*/
158 = Phase::ZERO;
159 bool mVsyncListening /*GUARDED_BY(mMutex)*/ = false;
160 };
161
RegionSamplingThread(SurfaceFlinger & flinger,Scheduler & scheduler,const TimingTunables & tunables)162 RegionSamplingThread::RegionSamplingThread(SurfaceFlinger& flinger, Scheduler& scheduler,
163 const TimingTunables& tunables)
164 : mFlinger(flinger),
165 mScheduler(scheduler),
166 mTunables(tunables),
167 mIdleTimer(std::chrono::duration_cast<std::chrono::milliseconds>(
168 mTunables.mSamplingTimerTimeout),
169 [] {}, [this] { checkForStaleLuma(); }),
170 mPhaseCallback(std::make_unique<SamplingOffsetCallback>(*this, mScheduler,
171 tunables.mSamplingOffset)),
172 lastSampleTime(0ns) {
__anon84e4d8880702() 173 mThread = std::thread([this]() { threadMain(); });
174 pthread_setname_np(mThread.native_handle(), "RegionSamplingThread");
175 mIdleTimer.start();
176 }
177
RegionSamplingThread(SurfaceFlinger & flinger,Scheduler & scheduler)178 RegionSamplingThread::RegionSamplingThread(SurfaceFlinger& flinger, Scheduler& scheduler)
179 : RegionSamplingThread(flinger, scheduler,
180 TimingTunables{defaultRegionSamplingOffset,
181 defaultRegionSamplingPeriod,
182 defaultRegionSamplingTimerTimeout}) {}
183
~RegionSamplingThread()184 RegionSamplingThread::~RegionSamplingThread() {
185 mIdleTimer.stop();
186
187 {
188 std::lock_guard lock(mThreadControlMutex);
189 mRunning = false;
190 mCondition.notify_one();
191 }
192
193 if (mThread.joinable()) {
194 mThread.join();
195 }
196 }
197
addListener(const Rect & samplingArea,const sp<IBinder> & stopLayerHandle,const sp<IRegionSamplingListener> & listener)198 void RegionSamplingThread::addListener(const Rect& samplingArea, const sp<IBinder>& stopLayerHandle,
199 const sp<IRegionSamplingListener>& listener) {
200 wp<Layer> stopLayer;
201 if (stopLayerHandle != nullptr && stopLayerHandle->localBinder() != nullptr) {
202 stopLayer = static_cast<Layer::Handle*>(stopLayerHandle.get())->owner;
203 }
204
205 sp<IBinder> asBinder = IInterface::asBinder(listener);
206 asBinder->linkToDeath(this);
207 std::lock_guard lock(mSamplingMutex);
208 mDescriptors.emplace(wp<IBinder>(asBinder), Descriptor{samplingArea, stopLayer, listener});
209 }
210
removeListener(const sp<IRegionSamplingListener> & listener)211 void RegionSamplingThread::removeListener(const sp<IRegionSamplingListener>& listener) {
212 std::lock_guard lock(mSamplingMutex);
213 mDescriptors.erase(wp<IBinder>(IInterface::asBinder(listener)));
214 }
215
checkForStaleLuma()216 void RegionSamplingThread::checkForStaleLuma() {
217 std::lock_guard lock(mThreadControlMutex);
218
219 if (mDiscardedFrames) {
220 ATRACE_INT(lumaSamplingStepTag, static_cast<int>(samplingStep::waitForZeroPhase));
221 mDiscardedFrames = false;
222 mPhaseCallback->startVsyncListener();
223 }
224 }
225
notifyNewContent()226 void RegionSamplingThread::notifyNewContent() {
227 doSample();
228 }
229
notifySamplingOffset()230 void RegionSamplingThread::notifySamplingOffset() {
231 doSample();
232 }
233
doSample()234 void RegionSamplingThread::doSample() {
235 std::lock_guard lock(mThreadControlMutex);
236 auto now = std::chrono::nanoseconds(systemTime(SYSTEM_TIME_MONOTONIC));
237 if (lastSampleTime + mTunables.mSamplingPeriod > now) {
238 ATRACE_INT(lumaSamplingStepTag, static_cast<int>(samplingStep::idleTimerWaiting));
239 mDiscardedFrames = true;
240 return;
241 }
242
243 ATRACE_INT(lumaSamplingStepTag, static_cast<int>(samplingStep::sample));
244
245 mDiscardedFrames = false;
246 lastSampleTime = now;
247
248 mIdleTimer.reset();
249 mPhaseCallback->stopVsyncListener();
250
251 mSampleRequested = true;
252 mCondition.notify_one();
253 }
254
binderDied(const wp<IBinder> & who)255 void RegionSamplingThread::binderDied(const wp<IBinder>& who) {
256 std::lock_guard lock(mSamplingMutex);
257 mDescriptors.erase(who);
258 }
259
260 namespace {
261 // Using Rec. 709 primaries
getLuma(float r,float g,float b)262 float getLuma(float r, float g, float b) {
263 constexpr auto rec709_red_primary = 0.2126f;
264 constexpr auto rec709_green_primary = 0.7152f;
265 constexpr auto rec709_blue_primary = 0.0722f;
266 return rec709_red_primary * r + rec709_green_primary * g + rec709_blue_primary * b;
267 }
268 } // anonymous namespace
269
sampleArea(const uint32_t * data,int32_t width,int32_t height,int32_t stride,uint32_t orientation,const Rect & sample_area)270 float sampleArea(const uint32_t* data, int32_t width, int32_t height, int32_t stride,
271 uint32_t orientation, const Rect& sample_area) {
272 if (!sample_area.isValid() || (sample_area.getWidth() > width) ||
273 (sample_area.getHeight() > height)) {
274 ALOGE("invalid sampling region requested");
275 return 0.0f;
276 }
277
278 // (b/133849373) ROT_90 screencap images produced upside down
279 auto area = sample_area;
280 if (orientation & ui::Transform::ROT_90) {
281 area.top = height - area.top;
282 area.bottom = height - area.bottom;
283 std::swap(area.top, area.bottom);
284
285 area.left = width - area.left;
286 area.right = width - area.right;
287 std::swap(area.left, area.right);
288 }
289
290 std::array<int32_t, 256> brightnessBuckets = {};
291 const int32_t majoritySampleNum = area.getWidth() * area.getHeight() / 2;
292
293 for (int32_t row = area.top; row < area.bottom; ++row) {
294 const uint32_t* rowBase = data + row * stride;
295 for (int32_t column = area.left; column < area.right; ++column) {
296 uint32_t pixel = rowBase[column];
297 const float r = (pixel & 0xFF) / 255.0f;
298 const float g = ((pixel >> 8) & 0xFF) / 255.0f;
299 const float b = ((pixel >> 16) & 0xFF) / 255.0f;
300 const uint8_t luma = std::round(getLuma(r, g, b) * 255.0f);
301 ++brightnessBuckets[luma];
302 if (brightnessBuckets[luma] > majoritySampleNum) return luma / 255.0f;
303 }
304 }
305
306 int32_t accumulated = 0;
307 size_t bucket = 0;
308 for (; bucket < brightnessBuckets.size(); bucket++) {
309 accumulated += brightnessBuckets[bucket];
310 if (accumulated > majoritySampleNum) break;
311 }
312
313 return bucket / 255.0f;
314 }
315
sampleBuffer(const sp<GraphicBuffer> & buffer,const Point & leftTop,const std::vector<RegionSamplingThread::Descriptor> & descriptors,uint32_t orientation)316 std::vector<float> RegionSamplingThread::sampleBuffer(
317 const sp<GraphicBuffer>& buffer, const Point& leftTop,
318 const std::vector<RegionSamplingThread::Descriptor>& descriptors, uint32_t orientation) {
319 void* data_raw = nullptr;
320 buffer->lock(GRALLOC_USAGE_SW_READ_OFTEN, &data_raw);
321 std::shared_ptr<uint32_t> data(reinterpret_cast<uint32_t*>(data_raw),
322 [&buffer](auto) { buffer->unlock(); });
323 if (!data) return {};
324
325 const int32_t width = buffer->getWidth();
326 const int32_t height = buffer->getHeight();
327 const int32_t stride = buffer->getStride();
328 std::vector<float> lumas(descriptors.size());
329 std::transform(descriptors.begin(), descriptors.end(), lumas.begin(),
330 [&](auto const& descriptor) {
331 return sampleArea(data.get(), width, height, stride, orientation,
332 descriptor.area - leftTop);
333 });
334 return lumas;
335 }
336
captureSample()337 void RegionSamplingThread::captureSample() {
338 ATRACE_CALL();
339 std::lock_guard lock(mSamplingMutex);
340
341 if (mDescriptors.empty()) {
342 return;
343 }
344
345 const auto device = mFlinger.getDefaultDisplayDevice();
346 const auto display = device->getCompositionDisplay();
347 const auto state = display->getState();
348 const auto orientation = static_cast<ui::Transform::orientation_flags>(state.orientation);
349
350 std::vector<RegionSamplingThread::Descriptor> descriptors;
351 Region sampleRegion;
352 for (const auto& [listener, descriptor] : mDescriptors) {
353 sampleRegion.orSelf(descriptor.area);
354 descriptors.emplace_back(descriptor);
355 }
356
357 const Rect sampledArea = sampleRegion.bounds();
358
359 auto dx = 0;
360 auto dy = 0;
361 switch (orientation) {
362 case ui::Transform::ROT_90:
363 dx = device->getWidth();
364 break;
365 case ui::Transform::ROT_180:
366 dx = device->getWidth();
367 dy = device->getHeight();
368 break;
369 case ui::Transform::ROT_270:
370 dy = device->getHeight();
371 break;
372 default:
373 break;
374 }
375
376 ui::Transform t(orientation);
377 auto screencapRegion = t.transform(sampleRegion);
378 screencapRegion = screencapRegion.translate(dx, dy);
379 DisplayRenderArea renderArea(device, screencapRegion.bounds(), sampledArea.getWidth(),
380 sampledArea.getHeight(), ui::Dataspace::V0_SRGB, orientation);
381
382 std::unordered_set<sp<IRegionSamplingListener>, SpHash<IRegionSamplingListener>> listeners;
383
384 auto traverseLayers = [&](const LayerVector::Visitor& visitor) {
385 bool stopLayerFound = false;
386 auto filterVisitor = [&](Layer* layer) {
387 // We don't want to capture any layers beyond the stop layer
388 if (stopLayerFound) return;
389
390 // Likewise if we just found a stop layer, set the flag and abort
391 for (const auto& [area, stopLayer, listener] : descriptors) {
392 if (layer == stopLayer.promote().get()) {
393 stopLayerFound = true;
394 return;
395 }
396 }
397
398 // Compute the layer's position on the screen
399 const Rect bounds = Rect(layer->getBounds());
400 const ui::Transform transform = layer->getTransform();
401 constexpr bool roundOutwards = true;
402 Rect transformed = transform.transform(bounds, roundOutwards);
403
404 // If this layer doesn't intersect with the larger sampledArea, skip capturing it
405 Rect ignore;
406 if (!transformed.intersect(sampledArea, &ignore)) return;
407
408 // If the layer doesn't intersect a sampling area, skip capturing it
409 bool intersectsAnyArea = false;
410 for (const auto& [area, stopLayer, listener] : descriptors) {
411 if (transformed.intersect(area, &ignore)) {
412 intersectsAnyArea = true;
413 listeners.insert(listener);
414 }
415 }
416 if (!intersectsAnyArea) return;
417
418 ALOGV("Traversing [%s] [%d, %d, %d, %d]", layer->getName().string(), bounds.left,
419 bounds.top, bounds.right, bounds.bottom);
420 visitor(layer);
421 };
422 mFlinger.traverseLayersInDisplay(device, filterVisitor);
423 };
424
425 sp<GraphicBuffer> buffer = nullptr;
426 if (mCachedBuffer && mCachedBuffer->getWidth() == sampledArea.getWidth() &&
427 mCachedBuffer->getHeight() == sampledArea.getHeight()) {
428 buffer = mCachedBuffer;
429 } else {
430 const uint32_t usage = GRALLOC_USAGE_SW_READ_OFTEN | GRALLOC_USAGE_HW_RENDER;
431 buffer = new GraphicBuffer(sampledArea.getWidth(), sampledArea.getHeight(),
432 PIXEL_FORMAT_RGBA_8888, 1, usage, "RegionSamplingThread");
433 }
434
435 bool ignored;
436 mFlinger.captureScreenCommon(renderArea, traverseLayers, buffer, false, ignored);
437
438 std::vector<Descriptor> activeDescriptors;
439 for (const auto& descriptor : descriptors) {
440 if (listeners.count(descriptor.listener) != 0) {
441 activeDescriptors.emplace_back(descriptor);
442 }
443 }
444
445 ALOGV("Sampling %zu descriptors", activeDescriptors.size());
446 std::vector<float> lumas =
447 sampleBuffer(buffer, sampledArea.leftTop(), activeDescriptors, orientation);
448 if (lumas.size() != activeDescriptors.size()) {
449 ALOGW("collected %zu median luma values for %zu descriptors", lumas.size(),
450 activeDescriptors.size());
451 return;
452 }
453
454 for (size_t d = 0; d < activeDescriptors.size(); ++d) {
455 activeDescriptors[d].listener->onSampleCollected(lumas[d]);
456 }
457
458 // Extend the lifetime of mCachedBuffer from the previous frame to here to ensure that:
459 // 1) The region sampling thread is the last owner of the buffer, and the freeing of the buffer
460 // happens in this thread, as opposed to the main thread.
461 // 2) The listener(s) receive their notifications prior to freeing the buffer.
462 mCachedBuffer = buffer;
463 ATRACE_INT(lumaSamplingStepTag, static_cast<int>(samplingStep::noWorkNeeded));
464 }
465
466 // NO_THREAD_SAFETY_ANALYSIS is because std::unique_lock presently lacks thread safety annotations.
threadMain()467 void RegionSamplingThread::threadMain() NO_THREAD_SAFETY_ANALYSIS {
468 std::unique_lock<std::mutex> lock(mThreadControlMutex);
469 while (mRunning) {
470 if (mSampleRequested) {
471 mSampleRequested = false;
472 lock.unlock();
473 captureSample();
474 lock.lock();
475 }
476 mCondition.wait(lock, [this]() REQUIRES(mThreadControlMutex) {
477 return mSampleRequested || !mRunning;
478 });
479 }
480 }
481
482 } // namespace android
483