1 /**
2 * Copyright (C) 2022 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include <ImsMediaVideoRenderer.h>
18 #include <ImsMediaTrace.h>
19 #include <ImsMediaTimer.h>
20 #include <ImsMediaVideoUtil.h>
21 #include <thread>
22
23 #define CODEC_TIMEOUT_NANO 100000
24 #define INTERVAL_MILLIS 10
25
ImsMediaVideoRenderer()26 ImsMediaVideoRenderer::ImsMediaVideoRenderer()
27 {
28 mCallback = nullptr;
29 mWindow = nullptr;
30 mCodec = nullptr;
31 mFormat = nullptr;
32 mCodecType = -1;
33 mWidth = 0;
34 mHeight = 0;
35 mFarOrientationDegree = 0;
36 mNearOrientationDegree = 0;
37 mStopped = false;
38 }
39
~ImsMediaVideoRenderer()40 ImsMediaVideoRenderer::~ImsMediaVideoRenderer()
41 {
42 while (!mFrameDatas.empty())
43 {
44 FrameData* frame = mFrameDatas.front();
45 delete frame;
46 mFrameDatas.pop_front();
47 }
48 }
49
SetSessionCallback(BaseSessionCallback * callback)50 void ImsMediaVideoRenderer::SetSessionCallback(BaseSessionCallback* callback)
51 {
52 mCallback = callback;
53 }
54
SetCodec(int32_t codecType)55 void ImsMediaVideoRenderer::SetCodec(int32_t codecType)
56 {
57 IMLOGD1("[SetCodec] codec[%d]", codecType);
58 mCodecType = codecType;
59 }
60
SetResolution(uint32_t width,uint32_t height)61 void ImsMediaVideoRenderer::SetResolution(uint32_t width, uint32_t height)
62 {
63 IMLOGD2("[SetResolution] width[%d], height[%d]", width, height);
64 mWidth = width;
65 mHeight = height;
66 }
67
SetDeviceOrientation(uint32_t orientation)68 void ImsMediaVideoRenderer::SetDeviceOrientation(uint32_t orientation)
69 {
70 IMLOGD1("[SetDeviceOrientation] orientation[%d]", orientation);
71 mNearOrientationDegree = orientation;
72 }
73
SetSurface(ANativeWindow * window)74 void ImsMediaVideoRenderer::SetSurface(ANativeWindow* window)
75 {
76 IMLOGD1("[SetSurface] surface[%p]", window);
77 mWindow = window;
78 }
79
Start()80 bool ImsMediaVideoRenderer::Start()
81 {
82 IMLOGD0("[Start]");
83 mMutex.lock();
84 mFormat = AMediaFormat_new();
85 AMediaFormat_setInt32(mFormat, AMEDIAFORMAT_KEY_WIDTH, mWidth);
86 AMediaFormat_setInt32(mFormat, AMEDIAFORMAT_KEY_HEIGHT, mHeight);
87
88 char kMimeType[128] = {'\0'};
89 sprintf(kMimeType, "video/avc");
90 if (mCodecType == kVideoCodecHevc)
91 {
92 sprintf(kMimeType, "video/hevc");
93 }
94
95 AMediaFormat_setString(mFormat, AMEDIAFORMAT_KEY_MIME, kMimeType);
96 AMediaFormat_setInt32(mFormat, AMEDIAFORMAT_KEY_COLOR_FORMAT,
97 21); // #21 : COLOR_FormatYUV420SemiPlanar
98 AMediaFormat_setInt32(mFormat, AMEDIAFORMAT_KEY_MAX_INPUT_SIZE, mWidth * mHeight * 10);
99 AMediaFormat_setInt32(mFormat, AMEDIAFORMAT_KEY_ROTATION, mFarOrientationDegree);
100
101 mCodec = AMediaCodec_createDecoderByType(kMimeType);
102 if (mCodec == nullptr)
103 {
104 IMLOGE0("[Start] Unable to create decoder");
105 return false;
106 }
107
108 if (mWindow != nullptr)
109 {
110 ANativeWindow_acquire(mWindow);
111 }
112
113 media_status_t err = AMediaCodec_configure(mCodec, mFormat, mWindow, nullptr, 0);
114 if (err != AMEDIA_OK)
115 {
116 IMLOGE1("[Start] configure error[%d]", err);
117 AMediaCodec_delete(mCodec);
118 mCodec = nullptr;
119 AMediaFormat_delete(mFormat);
120 mFormat = nullptr;
121 return false;
122 }
123
124 err = AMediaCodec_start(mCodec);
125 if (err != AMEDIA_OK)
126 {
127 IMLOGE1("[Start] codec start[%d]", err);
128 AMediaCodec_delete(mCodec);
129 mCodec = nullptr;
130 AMediaFormat_delete(mFormat);
131 mFormat = nullptr;
132 return false;
133 }
134
135 mStopped = false;
136 mMutex.unlock();
137 std::thread t1(&ImsMediaVideoRenderer::processBuffers, this);
138 t1.detach();
139 return true;
140 }
141
Stop()142 void ImsMediaVideoRenderer::Stop()
143 {
144 IMLOGD0("[Stop]");
145
146 mMutex.lock();
147 mStopped = true;
148 mMutex.unlock();
149 mConditionExit.wait_timeout(MAX_WAIT_RESTART);
150
151 if (mCodec != nullptr)
152 {
153 AMediaCodec_signalEndOfInputStream(mCodec);
154 AMediaCodec_stop(mCodec);
155 AMediaCodec_delete(mCodec);
156 mCodec = nullptr;
157 }
158
159 if (mWindow != nullptr)
160 {
161 ANativeWindow_release(mWindow);
162 }
163
164 if (mFormat != nullptr)
165 {
166 AMediaFormat_delete(mFormat);
167 mFormat = nullptr;
168 }
169 }
170
OnDataFrame(uint8_t * buffer,uint32_t size,uint32_t timestamp,const bool isConfigFrame)171 void ImsMediaVideoRenderer::OnDataFrame(
172 uint8_t* buffer, uint32_t size, uint32_t timestamp, const bool isConfigFrame)
173 {
174 if (size == 0 || buffer == nullptr)
175 {
176 return;
177 }
178
179 IMLOGD_PACKET2(IM_PACKET_LOG_VIDEO, "[OnDataFrame] frame size[%u], list[%d]", size,
180 mFrameDatas.size());
181 std::lock_guard<std::mutex> guard(mMutex);
182 if (mCodec == nullptr)
183 {
184 return;
185 }
186
187 mFrameDatas.push_back(new FrameData(buffer, size, timestamp, isConfigFrame));
188 }
189
processBuffers()190 void ImsMediaVideoRenderer::processBuffers()
191 {
192 uint32_t nextTime = ImsMediaTimer::GetTimeInMilliSeconds();
193 uint32_t timeDiff = 0;
194
195 IMLOGD1("[processBuffers] enter time[%u]", nextTime);
196
197 while (true)
198 {
199 mMutex.lock();
200 if (mStopped)
201 {
202 mMutex.unlock();
203 break;
204 }
205 mMutex.unlock();
206
207 if (mFrameDatas.size() == 0)
208 {
209 continue;
210 }
211
212 auto index = AMediaCodec_dequeueInputBuffer(mCodec, CODEC_TIMEOUT_NANO);
213
214 if (index >= 0)
215 {
216 size_t bufferSize = 0;
217 uint8_t* inputBuffer = AMediaCodec_getInputBuffer(mCodec, index, &bufferSize);
218
219 if (inputBuffer != nullptr)
220 {
221 FrameData* frame = mFrameDatas.front();
222 memcpy(inputBuffer, frame->data, frame->size);
223 IMLOGD_PACKET4(IM_PACKET_LOG_VIDEO,
224 "[processBuffers] queue input buffer index[%d], size[%d], TS[%d], "
225 "config[%d]",
226 index, frame->size, frame->timestamp, frame->isConfig);
227
228 media_status_t err = AMediaCodec_queueInputBuffer(
229 mCodec, index, 0, frame->size, frame->timestamp * 1000, 0);
230
231 if (err != AMEDIA_OK)
232 {
233 IMLOGE1("[processBuffers] Unable to queue input buffers - err[%d]", err);
234 }
235
236 delete frame;
237 mFrameDatas.pop_front();
238 }
239 }
240
241 AMediaCodecBufferInfo info;
242 index = AMediaCodec_dequeueOutputBuffer(mCodec, &info, CODEC_TIMEOUT_NANO);
243
244 if (index >= 0)
245 {
246 IMLOGD_PACKET5(IM_PACKET_LOG_VIDEO,
247 "[processBuffers] index[%d], size[%d], offset[%d], time[%ld], flags[%d]", index,
248 info.size, info.offset, info.presentationTimeUs, info.flags);
249
250 AMediaCodec_releaseOutputBuffer(mCodec, index, true);
251 }
252 else if (index == AMEDIACODEC_INFO_OUTPUT_BUFFERS_CHANGED)
253 {
254 IMLOGD0("[processBuffers] output buffer changed");
255 }
256 else if (index == AMEDIACODEC_INFO_OUTPUT_FORMAT_CHANGED)
257 {
258 if (mFormat != nullptr)
259 {
260 AMediaFormat_delete(mFormat);
261 }
262 mFormat = AMediaCodec_getOutputFormat(mCodec);
263 IMLOGD1("[processBuffers] format changed, format[%s]", AMediaFormat_toString(mFormat));
264 }
265 else if (index == AMEDIACODEC_INFO_TRY_AGAIN_LATER)
266 {
267 IMLOGD0("[processBuffers] no output buffer");
268 }
269 else
270 {
271 IMLOGD1("[processBuffers] unexpected index[%d]", index);
272 }
273
274 nextTime += INTERVAL_MILLIS;
275 uint32_t nCurrTime = ImsMediaTimer::GetTimeInMilliSeconds();
276
277 if (nextTime > nCurrTime)
278 {
279 timeDiff = nextTime - nCurrTime;
280 IMLOGD_PACKET1(IM_PACKET_LOG_VIDEO, "[processBuffers] timeDiff[%u]", timeDiff);
281 ImsMediaTimer::Sleep(timeDiff);
282 }
283 }
284
285 mConditionExit.signal();
286 IMLOGD0("[processBuffers] exit");
287 }
288
UpdateDeviceOrientation(uint32_t degree)289 void ImsMediaVideoRenderer::UpdateDeviceOrientation(uint32_t degree)
290 {
291 IMLOGD1("[UpdateDeviceOrientation] orientation[%d]", degree);
292 mNearOrientationDegree = degree;
293 }
294
UpdatePeerOrientation(uint32_t degree)295 void ImsMediaVideoRenderer::UpdatePeerOrientation(uint32_t degree)
296 {
297 IMLOGD1("[UpdatePeerOrientation] orientation[%d]", degree);
298
299 if (mFarOrientationDegree != degree)
300 {
301 Stop();
302 mFarOrientationDegree = degree;
303 Start();
304 }
305 }