1 /*
2 * Copyright (C) 2011 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16 #include "RenderThread.h"
17
18 #include "ChannelStream.h"
19 #include "FrameBuffer.h"
20 #include "ReadBuffer.h"
21 #include "RenderChannelImpl.h"
22 #include "RenderControl.h"
23 #include "RenderThreadInfo.h"
24 #include "RingStream.h"
25 #include "VkDecoderContext.h"
26 #include "apigen-codec-common/ChecksumCalculatorThreadInfo.h"
27 #include "aemu/base/HealthMonitor.h"
28 #include "aemu/base/synchronization/Lock.h"
29 #include "aemu/base/synchronization/MessageChannel.h"
30 #include "aemu/base/Metrics.h"
31 #include "aemu/base/files/StreamSerializing.h"
32 #include "aemu/base/system/System.h"
33 #include "host-common/feature_control.h"
34 #include "host-common/logging.h"
35 #include "vulkan/VkCommonOperations.h"
36
37 #define EMUGL_DEBUG_LEVEL 0
38 #include "host-common/debug.h"
39
40 #ifndef _WIN32
41 #include <unistd.h>
42 #endif
43
44 #include <assert.h>
45 #include <string.h>
46
47 #include <unordered_map>
48
49 namespace gfxstream {
50
51 using android::base::AutoLock;
52 using android::base::EventHangMetadata;
53 using android::base::MessageChannel;
54 using emugl::GfxApiLogger;
55 using vk::VkDecoderContext;
56
57 struct RenderThread::SnapshotObjects {
58 RenderThreadInfo* threadInfo;
59 ChecksumCalculator* checksumCalc;
60 ChannelStream* channelStream;
61 RingStream* ringStream;
62 ReadBuffer* readBuffer;
63 };
64
getBenchmarkEnabledFromEnv()65 static bool getBenchmarkEnabledFromEnv() {
66 auto threadEnabled = android::base::getEnvironmentVariable("ANDROID_EMUGL_RENDERTHREAD_STATS");
67 if (threadEnabled == "1") return true;
68 return false;
69 }
70
71 // Start with a smaller buffer to not waste memory on a low-used render threads.
72 static constexpr int kStreamBufferSize = 128 * 1024;
73
74 // Requires this many threads on the system available to run unlimited.
75 static constexpr int kMinThreadsToRunUnlimited = 5;
76
77 // A thread run limiter that limits render threads to run one slice at a time.
78 static android::base::Lock sThreadRunLimiter;
79
RenderThread(RenderChannelImpl * channel,android::base::Stream * loadStream)80 RenderThread::RenderThread(RenderChannelImpl* channel,
81 android::base::Stream* loadStream)
82 : android::base::Thread(android::base::ThreadFlags::MaskSignals, 2 * 1024 * 1024),
83 mChannel(channel),
84 mRunInLimitedMode(android::base::getCpuCoreCount() < kMinThreadsToRunUnlimited)
85 {
86 if (loadStream) {
87 const bool success = loadStream->getByte();
88 if (success) {
89 mStream.emplace(0);
90 android::base::loadStream(loadStream, &*mStream);
91 mState = SnapshotState::StartLoading;
92 } else {
93 mFinished.store(true, std::memory_order_relaxed);
94 }
95 }
96 }
97
RenderThread(struct asg_context context,android::base::Stream * loadStream,android::emulation::asg::ConsumerCallbacks callbacks,uint32_t contextId,uint32_t capsetId,std::optional<std::string> nameOpt)98 RenderThread::RenderThread(
99 struct asg_context context,
100 android::base::Stream* loadStream,
101 android::emulation::asg::ConsumerCallbacks callbacks,
102 uint32_t contextId, uint32_t capsetId,
103 std::optional<std::string> nameOpt)
104 : android::base::Thread(android::base::ThreadFlags::MaskSignals, 2 * 1024 * 1024,
105 std::move(nameOpt)),
106 mRingStream(
107 new RingStream(context, callbacks, kStreamBufferSize)),
108 mContextId(contextId), mCapsetId(capsetId) {
109 if (loadStream) {
110 const bool success = loadStream->getByte();
111 if (success) {
112 mStream.emplace(0);
113 android::base::loadStream(loadStream, &*mStream);
114 mState = SnapshotState::StartLoading;
115 } else {
116 mFinished.store(true, std::memory_order_relaxed);
117 }
118 }
119 }
120
121 // Note: the RenderThread destructor might be called from a different thread
122 // than from RenderThread::main() so thread specific cleanup likely belongs at
123 // the end of RenderThread::main().
124 RenderThread::~RenderThread() = default;
125
pausePreSnapshot()126 void RenderThread::pausePreSnapshot() {
127 AutoLock lock(mLock);
128 assert(mState == SnapshotState::Empty);
129 mStream.emplace();
130 mState = SnapshotState::StartSaving;
131 if (mRingStream) {
132 mRingStream->pausePreSnapshot();
133 // mCondVar.broadcastAndUnlock(&lock);
134 }
135 if (mChannel) {
136 mChannel->pausePreSnapshot();
137 mCondVar.broadcastAndUnlock(&lock);
138 }
139 }
140
resume()141 void RenderThread::resume() {
142 AutoLock lock(mLock);
143 // This function can be called for a thread from pre-snapshot loading
144 // state; it doesn't need to do anything.
145 if (mState == SnapshotState::Empty) {
146 return;
147 }
148 if (mRingStream) mRingStream->resume();
149 waitForSnapshotCompletion(&lock);
150 mStream.clear();
151 mState = SnapshotState::Empty;
152 if (mChannel) mChannel->resume();
153 if (mRingStream) mRingStream->resume();
154 mCondVar.broadcastAndUnlock(&lock);
155 }
156
save(android::base::Stream * stream)157 void RenderThread::save(android::base::Stream* stream) {
158 bool success;
159 {
160 AutoLock lock(mLock);
161 assert(mState == SnapshotState::StartSaving ||
162 mState == SnapshotState::InProgress ||
163 mState == SnapshotState::Finished);
164 waitForSnapshotCompletion(&lock);
165 success = mState == SnapshotState::Finished;
166 }
167
168 if (success) {
169 assert(mStream);
170 stream->putByte(1);
171 android::base::saveStream(stream, *mStream);
172 } else {
173 stream->putByte(0);
174 }
175 }
176
waitForSnapshotCompletion(AutoLock * lock)177 void RenderThread::waitForSnapshotCompletion(AutoLock* lock) {
178 while (mState != SnapshotState::Finished &&
179 !mFinished.load(std::memory_order_relaxed)) {
180 mCondVar.wait(lock);
181 }
182 }
183
184 template <class OpImpl>
snapshotOperation(AutoLock * lock,OpImpl && implFunc)185 void RenderThread::snapshotOperation(AutoLock* lock, OpImpl&& implFunc) {
186 assert(isPausedForSnapshotLocked());
187 mState = SnapshotState::InProgress;
188 mCondVar.broadcastAndUnlock(lock);
189
190 implFunc();
191
192 lock->lock();
193
194 mState = SnapshotState::Finished;
195 mCondVar.broadcast();
196
197 // Only return after we're allowed to proceed.
198 while (isPausedForSnapshotLocked()) {
199 mCondVar.wait(lock);
200 }
201 }
202
loadImpl(AutoLock * lock,const SnapshotObjects & objects)203 void RenderThread::loadImpl(AutoLock* lock, const SnapshotObjects& objects) {
204 snapshotOperation(lock, [this, &objects] {
205 objects.readBuffer->onLoad(&*mStream);
206 if (objects.channelStream) objects.channelStream->load(&*mStream);
207 if (objects.ringStream) objects.ringStream->load(&*mStream);
208 objects.checksumCalc->load(&*mStream);
209 objects.threadInfo->onLoad(&*mStream);
210 });
211 }
212
saveImpl(AutoLock * lock,const SnapshotObjects & objects)213 void RenderThread::saveImpl(AutoLock* lock, const SnapshotObjects& objects) {
214 snapshotOperation(lock, [this, &objects] {
215 objects.readBuffer->onSave(&*mStream);
216 if (objects.channelStream) objects.channelStream->save(&*mStream);
217 if (objects.ringStream) objects.ringStream->save(&*mStream);
218 objects.checksumCalc->save(&*mStream);
219 objects.threadInfo->onSave(&*mStream);
220 });
221 }
222
isPausedForSnapshotLocked() const223 bool RenderThread::isPausedForSnapshotLocked() const {
224 return mState != SnapshotState::Empty;
225 }
226
doSnapshotOperation(const SnapshotObjects & objects,SnapshotState state)227 bool RenderThread::doSnapshotOperation(const SnapshotObjects& objects,
228 SnapshotState state) {
229 AutoLock lock(mLock);
230 if (mState == state) {
231 switch (state) {
232 case SnapshotState::StartLoading:
233 loadImpl(&lock, objects);
234 return true;
235 case SnapshotState::StartSaving:
236 saveImpl(&lock, objects);
237 return true;
238 default:
239 return false;
240 }
241 }
242 return false;
243 }
244
setFinished()245 void RenderThread::setFinished() {
246 // Make sure it never happens that we wait forever for the thread to
247 // save to snapshot while it was not even going to.
248 AutoLock lock(mLock);
249 mFinished.store(true, std::memory_order_relaxed);
250 if (mState != SnapshotState::Empty) {
251 mCondVar.broadcastAndUnlock(&lock);
252 }
253 }
254
main()255 intptr_t RenderThread::main() {
256 if (mFinished.load(std::memory_order_relaxed)) {
257 ERR("Error: fail loading a RenderThread @%p", this);
258 return 0;
259 }
260
261 RenderThreadInfo tInfo;
262 ChecksumCalculatorThreadInfo tChecksumInfo;
263 ChecksumCalculator& checksumCalc = tChecksumInfo.get();
264 bool needRestoreFromSnapshot = false;
265
266 //
267 // initialize decoders
268 //
269 if (!feature_is_enabled(kFeature_GuestUsesAngle)) {
270 tInfo.initGl();
271 }
272
273 initRenderControlContext(&tInfo.m_rcDec);
274
275 if (!mChannel && !mRingStream) {
276 GL_LOG("Exited a loader RenderThread @%p", this);
277 mFinished.store(true, std::memory_order_relaxed);
278 return 0;
279 }
280
281 ChannelStream stream(mChannel, RenderChannel::Buffer::kSmallSize);
282 IOStream* ioStream =
283 mChannel ? (IOStream*)&stream : (IOStream*)mRingStream.get();
284
285 ReadBuffer readBuf(kStreamBufferSize);
286 if (mRingStream) {
287 readBuf.setNeededFreeTailSize(0);
288 }
289
290 const SnapshotObjects snapshotObjects = {
291 &tInfo, &checksumCalc, &stream, mRingStream.get(), &readBuf,
292 };
293
294 // Framebuffer initialization is asynchronous, so we need to make sure
295 // it's completely initialized before running any GL commands.
296 FrameBuffer::waitUntilInitialized();
297 if (vk::getGlobalVkEmulation()) {
298 tInfo.m_vkInfo.emplace();
299 }
300
301 tInfo.m_magmaInfo.emplace();
302
303 // This is the only place where we try loading from snapshot.
304 // But the context bind / restoration will be delayed after receiving
305 // the first GL command.
306 if (doSnapshotOperation(snapshotObjects, SnapshotState::StartLoading)) {
307 GL_LOG("Loaded RenderThread @%p from snapshot", this);
308 needRestoreFromSnapshot = true;
309 } else {
310 // Not loading from a snapshot: continue regular startup, read
311 // the |flags|.
312 uint32_t flags = 0;
313 while (ioStream->read(&flags, sizeof(flags)) != sizeof(flags)) {
314 // Stream read may fail because of a pending snapshot.
315 if (!doSnapshotOperation(snapshotObjects, SnapshotState::StartSaving)) {
316 setFinished();
317 GL_LOG("Exited a RenderThread @%p early", this);
318 return 0;
319 }
320 }
321
322 // |flags| used to mean something, now they're not used.
323 (void)flags;
324 }
325
326 int stats_totalBytes = 0;
327 uint64_t stats_progressTimeUs = 0;
328 auto stats_t0 = android::base::getHighResTimeUs() / 1000;
329 bool benchmarkEnabled = getBenchmarkEnabledFromEnv();
330
331 //
332 // open dump file if RENDER_DUMP_DIR is defined
333 //
334 const char* dump_dir = getenv("RENDERER_DUMP_DIR");
335 FILE* dumpFP = nullptr;
336 if (dump_dir) {
337 // size_t bsize = strlen(dump_dir) + 32;
338 // char* fname = new char[bsize];
339 // snprintf(fname, bsize, "%s" PATH_SEP "stream_%p", dump_dir, this);
340 // dumpFP = android_fopen(fname, "wb");
341 // if (!dumpFP) {
342 // fprintf(stderr, "Warning: stream dump failed to open file %s\n",
343 // fname);
344 // }
345 // delete[] fname;
346 }
347
348 GfxApiLogger gfxLogger;
349 auto& metricsLogger = FrameBuffer::getFB()->getMetricsLogger();
350
351 const ProcessResources* processResources = nullptr;
352
353 while (true) {
354 // Let's make sure we read enough data for at least some processing.
355 uint32_t packetSize;
356 if (readBuf.validData() >= 8) {
357 // We know that packet size is the second int32_t from the start.
358 packetSize = *(uint32_t*)(readBuf.buf() + 4);
359 if (!packetSize) {
360 // Emulator will get live-stuck here if packet size is read to be zero;
361 // crash right away so we can see these events.
362 // emugl::emugl_crash_reporter(
363 // "Guest should never send a size-0 GL packet\n");
364 }
365 } else {
366 // Read enough data to at least be able to get the packet size next
367 // time.
368 packetSize = 8;
369 }
370
371 int stat = 0;
372 if (packetSize > readBuf.validData()) {
373 stat = readBuf.getData(ioStream, packetSize);
374 if (stat <= 0) {
375 if (doSnapshotOperation(snapshotObjects, SnapshotState::StartSaving)) {
376 continue;
377 } else {
378 D("Warning: render thread could not read data from stream");
379 break;
380 }
381 } else if (needRestoreFromSnapshot) {
382 // If we're using RingStream that might load before FrameBuffer
383 // restores the contexts from the handles, so check again here.
384
385 tInfo.postLoadRefreshCurrentContextSurfacePtrs();
386 needRestoreFromSnapshot = false;
387 }
388 }
389
390 DD("render thread read %i bytes, op %i, packet size %i",
391 readBuf.validData(), *(uint32_t*)readBuf.buf(),
392 *(uint32_t*)(readBuf.buf() + 4));
393
394 //
395 // log received bandwidth statistics
396 //
397 if (benchmarkEnabled) {
398 stats_totalBytes += readBuf.validData();
399 auto dt = android::base::getHighResTimeUs() / 1000 - stats_t0;
400 if (dt > 1000) {
401 float dts = (float)dt / 1000.0f;
402 printf("Used Bandwidth %5.3f MB/s, time in progress %f ms total %f ms\n", ((float)stats_totalBytes / dts) / (1024.0f*1024.0f),
403 stats_progressTimeUs / 1000.0f,
404 (float)dt);
405 readBuf.printStats();
406 stats_t0 = android::base::getHighResTimeUs() / 1000;
407 stats_progressTimeUs = 0;
408 stats_totalBytes = 0;
409 }
410 }
411
412 //
413 // dump stream to file if needed
414 //
415 if (dumpFP) {
416 int skip = readBuf.validData() - stat;
417 fwrite(readBuf.buf() + skip, 1, readBuf.validData() - skip, dumpFP);
418 fflush(dumpFP);
419 }
420
421 bool progress;
422
423 do {
424 std::unique_ptr<EventHangMetadata::HangAnnotations> renderThreadData =
425 std::make_unique<EventHangMetadata::HangAnnotations>();
426
427 const char* processName = nullptr;
428 if (tInfo.m_processName) {
429 processName = tInfo.m_processName.value().c_str();
430 }
431
432 auto* healthMonitor = FrameBuffer::getFB()->getHealthMonitor();
433 if (healthMonitor) {
434 if (processName) {
435 renderThreadData->insert(
436 {{"renderthread_guest_process", processName}});
437 }
438 if (readBuf.validData() >= 4) {
439 renderThreadData->insert(
440 {{"first_opcode", std::to_string(*(uint32_t*)readBuf.buf())},
441 {"buffer_length", std::to_string(readBuf.validData())}});
442 }
443 }
444 auto watchdog = WATCHDOG_BUILDER(healthMonitor, "RenderThread decode operation")
445 .setHangType(EventHangMetadata::HangType::kRenderThread)
446 .setAnnotations(std::move(renderThreadData))
447 .build();
448
449 if (!processResources && tInfo.m_puid) {
450 processResources = FrameBuffer::getFB()->getProcessResources(tInfo.m_puid);
451 }
452
453 progress = false;
454 size_t last;
455
456 //
457 // try to process some of the command buffer using the
458 // Vulkan decoder
459 //
460 // Note: It's risky to limit Vulkan decoding to one thread,
461 // so we do it outside the limiter
462 if (tInfo.m_vkInfo) {
463 VkDecoderContext context = {
464 .processName = processName,
465 .gfxApiLogger = &gfxLogger,
466 .healthMonitor = FrameBuffer::getFB()->getHealthMonitor(),
467 .metricsLogger = &metricsLogger,
468 };
469 last = tInfo.m_vkInfo->m_vkDec.decode(readBuf.buf(), readBuf.validData(), ioStream,
470 processResources, context);
471 if (last > 0) {
472 if (!processResources) {
473 ERR("Processed some Vulkan packets without process resources created. "
474 "That's problematic.");
475 }
476 readBuf.consume(last);
477 progress = true;
478 }
479 }
480
481 if (mRunInLimitedMode) {
482 sThreadRunLimiter.lock();
483 }
484
485 // try to process some of the command buffer using the GLESv1
486 // decoder
487 //
488 // DRIVER WORKAROUND:
489 // On Linux with NVIDIA GPU's at least, we need to avoid performing
490 // GLES ops while someone else holds the FrameBuffer write lock.
491 //
492 // To be more specific, on Linux with NVIDIA Quadro K2200 v361.xx,
493 // we get a segfault in the NVIDIA driver when glTexSubImage2D
494 // is called at the same time as glXMake(Context)Current.
495 //
496 // To fix, this driver workaround avoids calling
497 // any sort of GLES call when we are creating/destroying EGL
498 // contexts.
499 {
500 FrameBuffer::getFB()->lockContextStructureRead();
501 }
502
503 if (tInfo.m_glInfo) {
504 {
505 last = tInfo.m_glInfo->m_glDec.decode(
506 readBuf.buf(), readBuf.validData(), ioStream, &checksumCalc);
507 if (last > 0) {
508 progress = true;
509 readBuf.consume(last);
510 }
511 }
512
513 //
514 // try to process some of the command buffer using the GLESv2
515 // decoder
516 //
517 {
518 last = tInfo.m_glInfo->m_gl2Dec.decode(readBuf.buf(), readBuf.validData(),
519 ioStream, &checksumCalc);
520
521 if (last > 0) {
522 progress = true;
523 readBuf.consume(last);
524 }
525 }
526 }
527
528 FrameBuffer::getFB()->unlockContextStructureRead();
529 //
530 // try to process some of the command buffer using the
531 // renderControl decoder
532 //
533 {
534 last = tInfo.m_rcDec.decode(readBuf.buf(), readBuf.validData(),
535 ioStream, &checksumCalc);
536 if (last > 0) {
537 readBuf.consume(last);
538 progress = true;
539 }
540 }
541
542 //
543 // try to process some of the command buffer using the Magma
544 // decoder
545 //
546 if (tInfo.m_magmaInfo && tInfo.m_magmaInfo->m_magmaDec)
547 {
548 last = tInfo.m_magmaInfo->m_magmaDec->decode(readBuf.buf(), readBuf.validData(),
549 ioStream, &checksumCalc);
550 if (last > 0) {
551 readBuf.consume(last);
552 progress = true;
553 }
554 }
555
556 if (mRunInLimitedMode) {
557 sThreadRunLimiter.unlock();
558 }
559
560 } while (progress);
561 }
562
563 if (dumpFP) {
564 fclose(dumpFP);
565 }
566
567 if (tInfo.m_glInfo) {
568 FrameBuffer::getFB()->drainGlRenderThreadResources();
569 }
570
571 setFinished();
572
573 GL_LOG("Exited a RenderThread @%p", this);
574 return 0;
575 }
576
577 } // namespace gfxstream
578