1 /*
2 * Copyright (c) 2021 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #define GL_GLEXT_PROTOTYPES
17 #define EGL_EGLEXT_PROTOTYPES
18
19 #include "surface_image.h"
20
21 #include "securec.h"
22 #include "sandbox_utils.h"
23
24 #include <atomic>
25 #include <sync_fence.h>
26 #include <unistd.h>
27 #include <window.h>
28
29 #include <EGL/egl.h>
30 #include <EGL/eglext.h>
31 #include <GLES/gl.h>
32 #include <GLES/glext.h>
33
34 namespace OHOS {
35 namespace {
36 // Get a uniqueID in a process
GetProcessUniqueId()37 static int GetProcessUniqueId()
38 {
39 static std::atomic<int> g_counter { 0 };
40 return g_counter.fetch_add(1);
41 }
42 }
43
SurfaceImage(uint32_t textureId,uint32_t textureTarget)44 SurfaceImage::SurfaceImage(uint32_t textureId, uint32_t textureTarget)
45 : ConsumerSurface("SurfaceImage-" + std::to_string(GetRealPid()) + "-" + std::to_string(GetProcessUniqueId())),
46 textureId_(textureId),
47 textureTarget_(textureTarget),
48 updateSurfaceImage_(false),
49 eglDisplay_(EGL_NO_DISPLAY),
50 eglContext_(EGL_NO_CONTEXT),
51 currentSurfaceImage_(0),
52 currentSurfaceBuffer_(nullptr),
53 currentSurfaceBufferFence_(-1), // -1 invalid
54 currentTimeStamp_(0)
55 {
56 InitSurfaceImage();
57 }
58
~SurfaceImage()59 SurfaceImage::~SurfaceImage()
60 {
61 }
62
InitSurfaceImage()63 void SurfaceImage::InitSurfaceImage()
64 {
65 std::string name = "SurfaceImage-" + std::to_string(GetRealPid()) + "-" + std::to_string(GetProcessUniqueId());
66 auto ret = ConsumerSurface::Init();
67 BLOGI("surfaceimage init");
68 if (ret != SURFACE_ERROR_OK) {
69 BLOGE("init surfaceimage failed");
70 }
71 surfaceImageName_ = name;
72 }
73
SetDefaultSize(int32_t width,int32_t height)74 SurfaceError SurfaceImage::SetDefaultSize(int32_t width, int32_t height)
75 {
76 return ConsumerSurface::SetDefaultWidthAndHeight(width, height);
77 }
78
MatrixProduct(const std::array<float,16> & lMat,const std::array<float,16> & rMat)79 std::array<float, 16> SurfaceImage::MatrixProduct(const std::array<float, 16>& lMat, const std::array<float, 16>& rMat)
80 {
81 return std::array<float, 16> {lMat[0] * rMat[0] + lMat[4] * rMat[1] + lMat[8] * rMat[2] + lMat[12] * rMat[3],
82 lMat[1] * rMat[0] + lMat[5] * rMat[1] + lMat[9] * rMat[2] + lMat[13] * rMat[3],
83 lMat[2] * rMat[0] + lMat[6] * rMat[1] + lMat[10] * rMat[2] + lMat[14] * rMat[3],
84 lMat[3] * rMat[0] + lMat[7] * rMat[1] + lMat[11] * rMat[2] + lMat[15] * rMat[3],
85
86 lMat[0] * rMat[4] + lMat[4] * rMat[5] + lMat[8] * rMat[6] + lMat[12] * rMat[7],
87 lMat[1] * rMat[4] + lMat[5] * rMat[5] + lMat[9] * rMat[6] + lMat[13] * rMat[7],
88 lMat[2] * rMat[4] + lMat[6] * rMat[5] + lMat[10] * rMat[6] + lMat[14] * rMat[7],
89 lMat[3] * rMat[4] + lMat[7] * rMat[5] + lMat[11] * rMat[6] + lMat[15] * rMat[7],
90
91 lMat[0] * rMat[8] + lMat[4] * rMat[9] + lMat[8] * rMat[10] + lMat[12] * rMat[11],
92 lMat[1] * rMat[8] + lMat[5] * rMat[9] + lMat[9] * rMat[10] + lMat[13] * rMat[11],
93 lMat[2] * rMat[8] + lMat[6] * rMat[9] + lMat[10] * rMat[10] + lMat[14] * rMat[11],
94 lMat[3] * rMat[8] + lMat[7] * rMat[9] + lMat[11] * rMat[10] + lMat[15] * rMat[11],
95
96 lMat[0] * rMat[12] + lMat[4] * rMat[13] + lMat[8] * rMat[14] + lMat[12] * rMat[15],
97 lMat[1] * rMat[12] + lMat[5] * rMat[13] + lMat[9] * rMat[14] + lMat[13] * rMat[15],
98 lMat[2] * rMat[12] + lMat[6] * rMat[13] + lMat[10] * rMat[14] + lMat[14] * rMat[15],
99 lMat[3] * rMat[12] + lMat[7] * rMat[13] + lMat[11] * rMat[14] + lMat[15] * rMat[15]};
100 }
101
ComputeTransformMatrix()102 void SurfaceImage::ComputeTransformMatrix()
103 {
104 static const std::array<float, 16> rotate90 = {0, 1, 0, 0, -1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1};
105
106 std::array<float, TRANSFORM_MATRIX_ELE_COUNT> transformMatrix = {1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1};
107 float tx = 0.f;
108 float ty = 0.f;
109 float sx = 1.f;
110 float sy = 1.f;
111 if (currentTransformType_ == GraphicTransformType::GRAPHIC_ROTATE_90) {
112 transformMatrix = MatrixProduct(transformMatrix, rotate90);
113 }
114 float bufferWidth = currentSurfaceBuffer_->GetWidth();
115 float bufferHeight = currentSurfaceBuffer_->GetHeight();
116 if (currentCrop_.w < bufferWidth) {
117 tx = (float(currentCrop_.x) / bufferWidth);
118 sx = (float(currentCrop_.w) / bufferWidth);
119 }
120 if (currentCrop_.h < bufferHeight) {
121 ty = (float(bufferHeight - currentCrop_.y) / bufferHeight);
122 sy = (float(currentCrop_.h) / bufferHeight);
123 }
124 static const std::array<float, 16> cropMatrix = {sx, 0, 0, 0, 0, sy, 0, 0, 0, 0, 1, 0, tx, ty, 0, 1};
125 transformMatrix = MatrixProduct(cropMatrix, transformMatrix);
126
127 auto ret = memcpy_s(currentTransformMatrix_.data(), sizeof(transformMatrix),
128 transformMatrix.data(), sizeof(transformMatrix));
129 if (ret != EOK) {
130 BLOGE("ComputeTransformMatrix: transformMatrix memcpy_s failed");
131 }
132 }
133
UpdateSurfaceImage()134 SurfaceError SurfaceImage::UpdateSurfaceImage()
135 {
136 std::lock_guard<std::mutex> lockGuard(opMutex_);
137
138 // validate egl state
139 SurfaceError ret = ValidateEglState();
140 if (ret != SURFACE_ERROR_OK) {
141 return ret;
142 }
143
144 // acquire buffer
145 sptr<SurfaceBuffer> buffer = nullptr;
146 int32_t fence;
147 int64_t timestamp;
148 Rect damage;
149
150 ret = AcquireBuffer(buffer, fence, timestamp, damage);
151 if (ret != SURFACE_ERROR_OK) {
152 if (ret == SURFACE_ERROR_NO_BUFFER) {
153 glBindTexture(textureTarget_, textureId_);
154 BLOGE("AcquireBuffer no buffer");
155 } else {
156 BLOGE("AcquireBuffer failed");
157 }
158 return ret;
159 }
160
161 uint32_t seqNum = buffer->GetSeqNum();
162 BLOGI("seqNum %{public}d", seqNum);
163 EGLImageKHR img = imageCacheSeqs_[seqNum].eglImage_;
164 glBindTexture(textureTarget_, textureId_);
165 glEGLImageTargetTexture2DOES(textureTarget_, static_cast<GLeglImageOES>(img));
166
167 while (glGetError() != GL_NO_ERROR) {
168 BLOGE("glEGLImageTargetTexture2DOES error");
169 ret = SURFACE_ERROR_API_FAILED;
170 }
171
172 if (ret == SURFACE_ERROR_OK) {
173 ret = WaitReleaseEGLSync(eglDisplay_);
174 }
175
176 if (ret != SURFACE_ERROR_OK) {
177 ReleaseBuffer(buffer, -1);
178 return ret;
179 }
180
181 if (seqNum != currentSurfaceImage_) {
182 ret = ReleaseBuffer(currentSurfaceBuffer_, -1);
183 if (ret != SURFACE_ERROR_OK) {
184 BLOGE("release currentSurfaceBuffer_ failed %{public}d", ret);
185 }
186 }
187
188 currentSurfaceImage_ = seqNum;
189 currentSurfaceBuffer_ = buffer;
190 currentSurfaceBufferFence_ = fence;
191 currentTimeStamp_ = timestamp;
192 currentCrop_ = damage;
193 currentTransformType_ = ConsumerSurface::GetTransform();
194
195 ComputeTransformMatrix();
196
197 ret = WaitOnFence();
198 return ret;
199 }
200
AttachContext(uint32_t textureId)201 SurfaceError SurfaceImage::AttachContext(uint32_t textureId)
202 {
203 std::lock_guard<std::mutex> lockGuard(opMutex_);
204 if (isAttached) {
205 BLOGI("SurfaceImage is already attached");
206 return SURFACE_ERROR_OK;
207 }
208 if (imageCacheSeqs_[currentSurfaceImage_].eglImage_ == EGL_NO_IMAGE_KHR) {
209 BLOGE("AttachContext failed, no eglImage");
210 return SURFACE_ERROR_ERROR;
211 }
212
213 EGLImageKHR img = imageCacheSeqs_[currentSurfaceImage_].eglImage_;
214 EGLDisplay disp = eglGetCurrentDisplay();
215 EGLContext context = eglGetCurrentContext();
216 if (disp == EGL_NO_DISPLAY || context == EGL_NO_CONTEXT) {
217 BLOGE("AttachContext failed, EGLDisplay or EGLContext is invalid");
218 return SURFACE_ERROR_INIT;
219 }
220
221 glBindTexture(textureTarget_, textureId);
222 textureId_ = textureId;
223 glEGLImageTargetTexture2DOES(textureTarget_, static_cast<GLeglImageOES>(img));
224 eglDisplay_ = disp;
225 eglContext_ = context;
226
227 isAttached = true;
228 return SURFACE_ERROR_OK;
229 }
230
DetachContext()231 SurfaceError SurfaceImage::DetachContext()
232 {
233 std::lock_guard<std::mutex> lockGuard(opMutex_);
234 if (!isAttached) {
235 BLOGI("SurfaceImage is already detached");
236 return SURFACE_ERROR_OK;
237 }
238 EGLDisplay disp = eglGetCurrentDisplay();
239 EGLContext context = eglGetCurrentContext();
240
241 if ((eglDisplay_ != disp && eglDisplay_ != EGL_NO_DISPLAY) || (disp == EGL_NO_DISPLAY)) {
242 BLOGE("EGLDisplay is invalid, errno : 0x%{public}x", eglGetError());
243 return SURFACE_ERROR_INIT;
244 }
245 if ((eglContext_ != context && eglContext_ != EGL_NO_CONTEXT) || (context == EGL_NO_CONTEXT)) {
246 BLOGE("EGLContext is invalid, errno : 0x%{public}x", eglGetError());
247 return SURFACE_ERROR_INIT;
248 }
249
250 SurfaceError ret = WaitReleaseEGLSync(eglDisplay_);
251 if (ret == SURFACE_ERROR_OK) {
252 glDeleteTextures(1, &textureId_);
253 } else {
254 return ret;
255 }
256
257 eglDisplay_ = EGL_NO_DISPLAY;
258 eglContext_ = EGL_NO_CONTEXT;
259
260 isAttached = false;
261 return SURFACE_ERROR_OK;
262 }
263
GetTimeStamp()264 int64_t SurfaceImage::GetTimeStamp()
265 {
266 std::lock_guard<std::mutex> lockGuard(opMutex_);
267 return currentTimeStamp_;
268 }
269
GetTransformMatrix(float matrix[16])270 SurfaceError SurfaceImage::GetTransformMatrix(float matrix[16])
271 {
272 std::lock_guard<std::mutex> lockGuard(opMutex_);
273 auto ret = memcpy_s(matrix, sizeof(currentTransformMatrix_),
274 currentTransformMatrix_.data(), sizeof(currentTransformMatrix_));
275 if (ret != EOK) {
276 BLOGE("GetTransformMatrix: currentTransformMatrix_ memcpy_s failed");
277 return SURFACE_ERROR_ERROR;
278 }
279 return SURFACE_ERROR_OK;
280 }
281
AcquireBuffer(sptr<SurfaceBuffer> & buffer,int32_t & fence,int64_t & timestamp,Rect & damage)282 SurfaceError SurfaceImage::AcquireBuffer(sptr<SurfaceBuffer>& buffer, int32_t &fence,
283 int64_t ×tamp, Rect &damage)
284 {
285 SurfaceError ret = ConsumerSurface::AcquireBuffer(buffer, fence, timestamp, damage);
286 if (ret != SURFACE_ERROR_OK) {
287 BLOGE("AcquireBuffer error");
288 return ret;
289 }
290 // get seq num
291 uint32_t seqNum = buffer->GetSeqNum();
292
293 if (buffer != nullptr) {
294 if (imageCacheSeqs_[seqNum].eglImage_ != EGL_NO_IMAGE_KHR) {
295 eglDestroyImageKHR(eglDisplay_, imageCacheSeqs_[seqNum].eglImage_);
296 imageCacheSeqs_[seqNum].eglImage_ = EGL_NO_IMAGE_KHR;
297 }
298 }
299
300 // create image
301 if (imageCacheSeqs_[seqNum].eglImage_ == EGL_NO_IMAGE_KHR) {
302 EGLImageKHR eglImage = CreateEGLImage(eglDisplay_, buffer);
303 if (eglImage == EGL_NO_IMAGE_KHR) {
304 return SURFACE_ERROR_INIT;
305 }
306 imageCacheSeqs_[seqNum].eglImage_ = eglImage;
307 }
308 return SURFACE_ERROR_OK;
309 }
310
ReleaseBuffer(sptr<SurfaceBuffer> & buffer,int32_t fence)311 SurfaceError SurfaceImage::ReleaseBuffer(sptr<SurfaceBuffer>& buffer, int32_t fence)
312 {
313 SurfaceError error = ConsumerSurface::ReleaseBuffer(buffer, fence);
314 if (error != SURFACE_ERROR_OK) {
315 BLOGE("ReleaseBuffer error");
316 return error;
317 }
318 uint32_t seqNum = buffer->GetSeqNum();
319
320 imageCacheSeqs_[seqNum].eglSync_ = EGL_NO_SYNC_KHR;
321 return SURFACE_ERROR_OK;
322 }
323
ValidateEglState()324 SurfaceError SurfaceImage::ValidateEglState()
325 {
326 EGLDisplay disp = eglGetCurrentDisplay();
327 EGLContext context = eglGetCurrentContext();
328
329 if ((eglDisplay_ != disp && eglDisplay_ != EGL_NO_DISPLAY) || (disp == EGL_NO_DISPLAY)) {
330 BLOGE("EGLDisplay is invalid, errno : 0x%{public}x", eglGetError());
331 return SURFACE_ERROR_INIT;
332 }
333 if ((eglContext_ != context && eglContext_ != EGL_NO_CONTEXT) || (context == EGL_NO_CONTEXT)) {
334 BLOGE("EGLContext is invalid, errno : 0x%{public}x", eglGetError());
335 return SURFACE_ERROR_INIT;
336 }
337
338 eglDisplay_ = disp;
339 eglContext_ = context;
340 return SURFACE_ERROR_OK;
341 }
342
CreateEGLImage(EGLDisplay disp,const sptr<SurfaceBuffer> & buffer)343 EGLImageKHR SurfaceImage::CreateEGLImage(EGLDisplay disp, const sptr<SurfaceBuffer>& buffer)
344 {
345 sptr<SurfaceBuffer> bufferImpl = buffer;
346 NativeWindowBuffer* nBuffer = CreateNativeWindowBufferFromSurfaceBuffer(&bufferImpl);
347 EGLint attrs[] = {
348 EGL_IMAGE_PRESERVED,
349 EGL_TRUE,
350 EGL_NONE,
351 };
352
353 EGLImageKHR img = eglCreateImageKHR(disp, EGL_NO_CONTEXT, EGL_NATIVE_BUFFER_OHOS, nBuffer, attrs);
354 if (img == EGL_NO_IMAGE_KHR) {
355 EGLint error = eglGetError();
356 BLOGE("failed, error %{public}d", error);
357 eglTerminate(disp);
358 }
359 return img;
360 }
361
WaitReleaseEGLSync(EGLDisplay disp)362 SurfaceError SurfaceImage::WaitReleaseEGLSync(EGLDisplay disp)
363 {
364 // check fence extension
365 EGLSyncKHR fence = imageCacheSeqs_[currentSurfaceImage_].eglSync_;
366 if (fence != EGL_NO_SYNC_KHR) {
367 EGLint ret = eglClientWaitSyncKHR(disp, fence, 0, 1000000000);
368 if (ret == EGL_FALSE) {
369 BLOGE("eglClientWaitSyncKHR error 0x%{public}x", eglGetError());
370 return SURFACE_ERROR_ERROR;
371 } else if (ret == EGL_TIMEOUT_EXPIRED_KHR) {
372 BLOGE("eglClientWaitSyncKHR timeout");
373 return SURFACE_ERROR_ERROR;
374 }
375 eglDestroySyncKHR(disp, fence);
376 }
377 fence = eglCreateSyncKHR(disp, EGL_SYNC_FENCE_KHR, NULL);
378 if (fence == EGL_NO_SYNC_KHR) {
379 BLOGE("eglCreateSyncKHR error 0x%{public}x", eglGetError());
380 return SURFACE_ERROR_ERROR;
381 }
382 glFlush();
383 imageCacheSeqs_[currentSurfaceImage_].eglSync_ = fence;
384 return SURFACE_ERROR_OK;
385 }
386
WaitOnFence()387 SurfaceError SurfaceImage::WaitOnFence()
388 {
389 // check ret error code
390 EGLDisplay disp = eglGetCurrentDisplay();
391 EGLContext context = eglGetCurrentContext();
392 if ((eglDisplay_ != disp && eglDisplay_ != EGL_NO_DISPLAY) || (disp == EGL_NO_DISPLAY)) {
393 BLOGE("EGLDisplay is invalid, errno : 0x%{public}x", eglGetError());
394 return SURFACE_ERROR_INIT;
395 }
396 if ((eglContext_ != context && eglContext_ != EGL_NO_CONTEXT) || (context == EGL_NO_CONTEXT)) {
397 BLOGE("EGLContext is invalid, errno : 0x%{public}x", eglGetError());
398 return SURFACE_ERROR_INIT;
399 }
400
401 // check EGL_FENCE_KHR
402 if (currentSurfaceBufferFence_ != -1) {
403 BLOGE("currentSurfaceBufferFence_ fd - %{public}d", currentSurfaceBufferFence_);
404 sptr<SyncFence> fence = new SyncFence(currentSurfaceBufferFence_);
405 fence->Wait(-1);
406 }
407 return SURFACE_ERROR_OK;
408 }
409
~SurfaceImageListener()410 SurfaceImageListener::~SurfaceImageListener()
411 {
412 BLOGE("~SurfaceImageListener");
413 surfaceImage_ = nullptr;
414 }
415
OnBufferAvailable()416 void SurfaceImageListener::OnBufferAvailable()
417 {
418 BLOGE("SurfaceImageListener::OnBufferAvailable");
419 auto surfaceImage = surfaceImage_.promote();
420 if (surfaceImage == nullptr) {
421 BLOGE("surfaceImage promote failed");
422 return;
423 }
424
425 // check here maybe a messagequeue, flag instead now
426 surfaceImage->OnUpdateBufferAvailableState(true);
427 }
428 } // namespace OHOS
429