1 /*
2 * Copyright (c) 2022 HiSilicon (Shanghai) Technologies CO., LIMITED.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include <iostream>
17 #include <cstdio>
18 #include <cstdlib>
19 #include <cstring>
20 #include <csignal>
21 #include <unistd.h>
22 #include <pthread.h>
23 #include <sys/types.h>
24 #include <sys/stat.h>
25 #include <fcntl.h>
26 #include <sys/ioctl.h>
27 #include <sys/prctl.h>
28
29 #include "hi_mipi_tx.h"
30 #include "sdk.h"
31 #include "sample_comm.h"
32 #include "ai_infer_process.h"
33 #include "tennis_detect.h"
34 #include "vgs_img.h"
35 #include "base_interface.h"
36 #include "posix_help.h"
37 #include "sample_media_ai.h"
38 #include "sample_media_opencv.h"
39
40 using namespace std;
41
42 static HI_BOOL s_bOpenCVProcessStopSignal = HI_FALSE;
43 static pthread_t g_openCVProcessThread = 0;
44 static int g_opencv = 0;
45 static AicMediaInfo g_aicTennisMediaInfo = { 0 };
46 static AiPlugLib g_tennisWorkPlug = {0};
47 static HI_CHAR tennisDetectThreadName[16] = {0};
48
49 /*
50 * 设置VI设备信息
51 * Set VI device information
52 */
TennisViCfgSetDev(ViCfg * self,int devId,WDR_MODE_E wdrMode)53 static void TennisViCfgSetDev(ViCfg* self, int devId, WDR_MODE_E wdrMode)
54 {
55 HI_ASSERT(self);
56 HI_ASSERT((int)wdrMode < WDR_MODE_BUTT);
57
58 self->astViInfo[0].stDevInfo.ViDev = devId;
59 self->astViInfo[0].stDevInfo.enWDRMode = wdrMode;
60 }
61
62 /*
63 * 设置VI通道
64 * Set up the VI channel
65 */
TennisViCfgSetChn(ViCfg * self,int chnId,PIXEL_FORMAT_E pixFormat,VIDEO_FORMAT_E videoFormat,DYNAMIC_RANGE_E dynamicRange)66 static void TennisViCfgSetChn(ViCfg* self, int chnId, PIXEL_FORMAT_E pixFormat,
67 VIDEO_FORMAT_E videoFormat, DYNAMIC_RANGE_E dynamicRange)
68 {
69 HI_ASSERT(self);
70 HI_ASSERT((int)pixFormat < PIXEL_FORMAT_BUTT);
71 HI_ASSERT((int)videoFormat < VIDEO_FORMAT_BUTT);
72 HI_ASSERT((int)dynamicRange < DYNAMIC_RANGE_BUTT);
73
74 self->astViInfo[0].stChnInfo.ViChn = chnId;
75 self->astViInfo[0].stChnInfo.enPixFormat = pixFormat;
76 self->astViInfo[0].stChnInfo.enVideoFormat = videoFormat;
77 self->astViInfo[0].stChnInfo.enDynamicRange = dynamicRange;
78 }
79
TennisViPramCfg(HI_VOID)80 static HI_VOID TennisViPramCfg(HI_VOID)
81 {
82 ViCfgInit(&g_aicTennisMediaInfo.viCfg);
83 TennisViCfgSetDev(&g_aicTennisMediaInfo.viCfg, 0, WDR_MODE_NONE);
84 ViCfgSetPipe(&g_aicTennisMediaInfo.viCfg, 0, -1, -1, -1);
85 g_aicTennisMediaInfo.viCfg.astViInfo[0].stPipeInfo.enMastPipeMode = VI_OFFLINE_VPSS_OFFLINE;
86 TennisViCfgSetChn(&g_aicTennisMediaInfo.viCfg, 0, PIXEL_FORMAT_YVU_SEMIPLANAR_420,
87 VIDEO_FORMAT_LINEAR, DYNAMIC_RANGE_SDR8);
88 g_aicTennisMediaInfo.viCfg.astViInfo[0].stChnInfo.enCompressMode = COMPRESS_MODE_SEG;
89 }
90
TennisStVbParamCfg(VbCfg * self)91 static HI_VOID TennisStVbParamCfg(VbCfg *self)
92 {
93 memset_s(&g_aicTennisMediaInfo.vbCfg, sizeof(VB_CONFIG_S), 0, sizeof(VB_CONFIG_S));
94 // 2: The number of buffer pools that can be accommodated in the entire system
95 self->u32MaxPoolCnt = 2;
96
97 /*
98 * 获取一帧图片的buffer大小
99 * Get picture buffer size
100 */
101 g_aicTennisMediaInfo.u32BlkSize = COMMON_GetPicBufferSize(g_aicTennisMediaInfo.stSize.u32Width,
102 g_aicTennisMediaInfo.stSize.u32Height, SAMPLE_PIXEL_FORMAT, DATA_BITWIDTH_8, COMPRESS_MODE_SEG, DEFAULT_ALIGN);
103 self->astCommPool[0].u64BlkSize = g_aicTennisMediaInfo.u32BlkSize;
104 // 10: Number of cache blocks per cache pool. Value range: (0, 10240]
105 self->astCommPool[0].u32BlkCnt = 10;
106
107 /*
108 * 获取raw buffer的大小
109 * Get raw buffer size
110 */
111 g_aicTennisMediaInfo.u32BlkSize = VI_GetRawBufferSize(g_aicTennisMediaInfo.stSize.u32Width,
112 g_aicTennisMediaInfo.stSize.u32Height, PIXEL_FORMAT_RGB_BAYER_16BPP, COMPRESS_MODE_NONE, DEFAULT_ALIGN);
113 self->astCommPool[1].u64BlkSize = g_aicTennisMediaInfo.u32BlkSize;
114 // 4: Number of cache blocks per cache pool. Value range: (0, 10240]
115 self->astCommPool[1].u32BlkCnt = 4;
116 }
117
TennisVpssParamCfg(HI_VOID)118 static HI_VOID TennisVpssParamCfg(HI_VOID)
119 {
120 VpssCfgInit(&g_aicTennisMediaInfo.vpssCfg);
121 VpssCfgSetGrp(&g_aicTennisMediaInfo.vpssCfg, 0, NULL,
122 g_aicTennisMediaInfo.stSize.u32Width, g_aicTennisMediaInfo.stSize.u32Width);
123 g_aicTennisMediaInfo.vpssCfg.grpAttr.enPixelFormat = PIXEL_FORMAT_YVU_SEMIPLANAR_420;
124 // 1920:AICSTART_VI_OUTWIDTH, 1080: AICSTART_VI_OUTHEIGHT
125 VpssCfgAddChn(&g_aicTennisMediaInfo.vpssCfg, 1, NULL, 1920, 1080);
126 HI_ASSERT(!g_aicTennisMediaInfo.viSess);
127 }
128
TennisStVoParamCfg(VoCfg * self)129 static HI_VOID TennisStVoParamCfg(VoCfg *self)
130 {
131 SAMPLE_COMM_VO_GetDefConfig(self);
132 self->enDstDynamicRange = DYNAMIC_RANGE_SDR8;
133
134 self->enVoIntfType = VO_INTF_MIPI; /* set VO int type */
135 self->enIntfSync = VO_OUTPUT_USER; /* set VO output information */
136
137 self->enPicSize = g_aicTennisMediaInfo.enPicSize;
138 }
139
TennisDetectAiProcess(VIDEO_FRAME_INFO_S frm,VO_LAYER voLayer,VO_CHN voChn)140 static HI_VOID TennisDetectAiProcess(VIDEO_FRAME_INFO_S frm, VO_LAYER voLayer, VO_CHN voChn)
141 {
142 int ret;
143 tennis_detect opencv;
144 if (GetCfgBool("tennis_detect_switch:support_tennis_detect", true)) {
145 if (g_tennisWorkPlug.model == 0) {
146 ret = opencv.TennisDetectLoad(&g_tennisWorkPlug.model);
147 if (ret < 0) {
148 g_tennisWorkPlug.model = 0;
149 SAMPLE_CHECK_EXPR_GOTO(ret < 0, TENNIS_RELEASE, "TennisDetectLoad err, ret=%#x\n", ret);
150 }
151 }
152
153 VIDEO_FRAME_INFO_S calFrm;
154 ret = MppFrmResize(&frm, &calFrm, 640, 480); // 640: FRM_WIDTH, 480: FRM_HEIGHT
155 ret = opencv.TennisDetectCal(g_tennisWorkPlug.model, &calFrm, &frm);
156 SAMPLE_CHECK_EXPR_GOTO(ret < 0, TENNIS_RELEASE, "TennisDetectCal err, ret=%#x\n", ret);
157
158 ret = HI_MPI_VO_SendFrame(voLayer, voChn, &frm, 0);
159 SAMPLE_CHECK_EXPR_GOTO(ret != HI_SUCCESS, TENNIS_RELEASE,
160 "HI_MPI_VO_SendFrame err, ret=%#x\n", ret);
161 MppFrmDestroy(&calFrm);
162 }
163
164 TENNIS_RELEASE:
165 ret = HI_MPI_VPSS_ReleaseChnFrame(g_aicTennisMediaInfo.vpssGrp, g_aicTennisMediaInfo.vpssChn0, &frm);
166 if (ret != HI_SUCCESS) {
167 SAMPLE_PRT("Error(%#x),HI_MPI_VPSS_ReleaseChnFrame failed,Grp(%d) chn(%d)!\n",
168 ret, g_aicTennisMediaInfo.vpssGrp, g_aicTennisMediaInfo.vpssChn0);
169 }
170 }
171
GetVpssChnFrameTennisDetect(HI_VOID * arg)172 static HI_VOID* GetVpssChnFrameTennisDetect(HI_VOID* arg)
173 {
174 int ret;
175 VIDEO_FRAME_INFO_S frm;
176 HI_S32 s32MilliSec = 2000;
177 VO_LAYER voLayer = 0;
178 VO_CHN voChn = 0;
179
180 SAMPLE_PRT("vpssGrp:%d, vpssChn0:%d\n", g_aicTennisMediaInfo.vpssGrp, g_aicTennisMediaInfo.vpssChn0);
181
182 while (HI_FALSE == s_bOpenCVProcessStopSignal) {
183 ret = HI_MPI_VPSS_GetChnFrame(g_aicTennisMediaInfo.vpssGrp, g_aicTennisMediaInfo.vpssChn0, &frm, s32MilliSec);
184 if (ret != 0) {
185 SAMPLE_PRT("HI_MPI_VPSS_GetChnFrame FAIL, err=%#x, grp=%d, chn=%d\n",
186 ret, g_aicTennisMediaInfo.vpssGrp, g_aicTennisMediaInfo.vpssChn0);
187 ret = HI_MPI_VPSS_ReleaseChnFrame(g_aicTennisMediaInfo.vpssGrp, g_aicTennisMediaInfo.vpssChn0, &frm);
188 if (ret != HI_SUCCESS) {
189 SAMPLE_PRT("Error(%#x),HI_MPI_VPSS_ReleaseChnFrame failed,Grp(%d) chn(%d)!\n",
190 ret, g_aicTennisMediaInfo.vpssGrp, g_aicTennisMediaInfo.vpssChn0);
191 }
192 continue;
193 }
194 SAMPLE_PRT("get vpss frame success, weight:%d, height:%d\n", frm.stVFrame.u32Width, frm.stVFrame.u32Height);
195
196 if (g_opencv == 0) {
197 ConfBaseInit("./sample_ai.conf");
198 g_opencv++;
199 }
200 TennisDetectAiProcess(frm, voLayer, voChn);
201 }
202
203 return HI_NULL;
204 }
205
PauseDoUnloadTennisModel(HI_VOID)206 static HI_VOID PauseDoUnloadTennisModel(HI_VOID)
207 {
208 if (GetCfgBool("tennis_detect_switch:support_tennis_detect", true)) {
209 memset_s(&g_tennisWorkPlug, sizeof(g_tennisWorkPlug), 0x00, sizeof(g_tennisWorkPlug));
210 ConfBaseExt();
211 SAMPLE_PRT("tennis detect exit success\n");
212 g_opencv = 0;
213 }
214 }
215
TennisDetectAiThreadProcess(HI_VOID)216 static HI_S32 TennisDetectAiThreadProcess(HI_VOID)
217 {
218 HI_S32 s32Ret;
219 if (snprintf_s(tennisDetectThreadName, sizeof(tennisDetectThreadName),
220 sizeof(tennisDetectThreadName) - 1, "OpencvProcess") < 0) {
221 HI_ASSERT(0);
222 }
223 prctl(PR_SET_NAME, (unsigned long)tennisDetectThreadName, 0, 0, 0);
224 s32Ret = pthread_create(&g_openCVProcessThread, NULL, GetVpssChnFrameTennisDetect, NULL);
225
226 return s32Ret;
227 }
228
229 /*
230 * 将sensor采集到数据显示到液晶屏上，同时创建线程运行网球检测推理计算
231 * 视频输入->视频处理子系统->视频输出->显示屏
232 *
233 * Display the data collected by the sensor on the LCD screen,
234 * and at the same time create a thread to run tennis detect reasoning calculations
235 * VI->VPSS->VO->MIPI
236 */
SAMPLE_MEDIA_TENNIS_DETECT(HI_VOID)237 HI_S32 sample_media_opencv::SAMPLE_MEDIA_TENNIS_DETECT(HI_VOID)
238 {
239 HI_S32 s32Ret;
240 HI_S32 fd = 0;
241
242 /*
243 * 配置VI参数
244 * Config VI parameter
245 */
246 TennisViPramCfg();
247
248 /*
249 * 通过Sensor型号获取enPicSize
250 * Obtain enPicSize through the Sensor type
251 */
252 s32Ret = SAMPLE_COMM_VI_GetSizeBySensor(g_aicTennisMediaInfo.viCfg.astViInfo[0].stSnsInfo.enSnsType,
253 &g_aicTennisMediaInfo.enPicSize);
254 SAMPLE_CHECK_EXPR_RET(s32Ret != HI_SUCCESS, s32Ret, "get pic size by sensor fail, s32Ret=%#x\n", s32Ret);
255
256 /*
257 * 根据enPicSize，得到图片的宽高
258 * Get picture size(w*h), according enPicSize
259 */
260 s32Ret = SAMPLE_COMM_SYS_GetPicSize(g_aicTennisMediaInfo.enPicSize, &g_aicTennisMediaInfo.stSize);
261 SAMPLE_PRT("AIC: snsMaxSize=%ux%u\n", g_aicTennisMediaInfo.stSize.u32Width, g_aicTennisMediaInfo.stSize.u32Height);
262 SAMPLE_CHECK_EXPR_RET(s32Ret != HI_SUCCESS, s32Ret, "get picture size failed, s32Ret=%#x\n", s32Ret);
263
264 /*
265 * 配置VB参数
266 * Config VB parameter
267 */
268 TennisStVbParamCfg(&g_aicTennisMediaInfo.vbCfg);
269
270 /*
271 * 视频缓存池初始化以及MPI系统初始化
272 * VB init & MPI system init
273 */
274 s32Ret = SAMPLE_COMM_SYS_Init(&g_aicTennisMediaInfo.vbCfg);
275 SAMPLE_CHECK_EXPR_RET(s32Ret != HI_SUCCESS, s32Ret, "system init failed, s32Ret=%#x\n", s32Ret);
276
277 /*
278 * 设置VO至MIPI通路，获取MIPI设备
279 * Set VO config to MIPI, get MIPI device
280 */
281 s32Ret = SAMPLE_VO_CONFIG_MIPI(&fd);
282 SAMPLE_CHECK_EXPR_GOTO(s32Ret != HI_SUCCESS, EXIT, "CONFIG MIPI FAIL.s32Ret:0x%x\n", s32Ret);
283
284 /*
285 * 配置VPSS参数
286 * Config VPSS parameter
287 */
288 TennisVpssParamCfg();
289 s32Ret = ViVpssCreate(&g_aicTennisMediaInfo.viSess, &g_aicTennisMediaInfo.viCfg, &g_aicTennisMediaInfo.vpssCfg);
290 SAMPLE_CHECK_EXPR_GOTO(s32Ret != HI_SUCCESS, EXIT1, "ViVpss Sess create FAIL, ret=%#x\n", s32Ret);
291 g_aicTennisMediaInfo.vpssGrp = 0;
292 g_aicTennisMediaInfo.vpssChn0 = 1;
293
294 /*
295 * 配置VO参数
296 * Config VO parameter
297 */
298 TennisStVoParamCfg(&g_aicTennisMediaInfo.voCfg);
299
300 /*
301 * 启动VO到MIPI lcd通路
302 * Start VO to MIPI lcd
303 */
304 s32Ret = SampleCommVoStartMipi(&g_aicTennisMediaInfo.voCfg);
305 SAMPLE_CHECK_EXPR_GOTO(s32Ret != HI_SUCCESS, EXIT1, "start vo FAIL. s32Ret: 0x%x\n", s32Ret);
306
307 /*
308 * VPSS绑定VO
309 * VPSS bind VO
310 */
311 s32Ret = SAMPLE_COMM_VPSS_Bind_VO(g_aicTennisMediaInfo.vpssGrp,
312 g_aicTennisMediaInfo.vpssChn0, g_aicTennisMediaInfo.voCfg.VoDev, 0);
313 SAMPLE_CHECK_EXPR_GOTO(s32Ret != HI_SUCCESS, EXIT2, "vo bind vpss FAIL. s32Ret: 0x%x\n", s32Ret);
314 SAMPLE_PRT("vpssGrp:%d, vpssChn:%d\n", g_aicTennisMediaInfo.vpssGrp, g_aicTennisMediaInfo.vpssChn0);
315
316 /*
317 * 创建工作线程运行ai
318 * Create work thread to run ai
319 */
320 s32Ret = TennisDetectAiThreadProcess();
321 SAMPLE_CHECK_EXPR_RET(s32Ret != HI_SUCCESS, s32Ret, "ai proccess thread creat fail:%s\n", strerror(s32Ret));
322 PAUSE();
323 s_bOpenCVProcessStopSignal = HI_TRUE;
324 /*
325 * 等待一个线程结束，线程间同步的操作
326 * Waiting for the end of a thread, the operation of synchronization between threads
327 */
328 pthread_join(g_openCVProcessThread, nullptr);
329 g_openCVProcessThread = 0;
330 PauseDoUnloadTennisModel();
331
332 SAMPLE_COMM_VPSS_UnBind_VO(g_aicTennisMediaInfo.vpssGrp,
333 g_aicTennisMediaInfo.vpssChn0, g_aicTennisMediaInfo.voCfg.VoDev, 0);
334 SAMPLE_VO_DISABLE_MIPITx(fd);
335 SampleCloseMipiTxFd(fd);
336 system("echo 0 > /sys/class/gpio/gpio55/value");
337
338 EXIT2:
339 SAMPLE_COMM_VO_StopVO(&g_aicTennisMediaInfo.voCfg);
340 EXIT1:
341 VpssStop(&g_aicTennisMediaInfo.vpssCfg);
342 SAMPLE_COMM_VI_UnBind_VPSS(g_aicTennisMediaInfo.viCfg.astViInfo[0].stPipeInfo.aPipe[0],
343 g_aicTennisMediaInfo.viCfg.astViInfo[0].stChnInfo.ViChn, g_aicTennisMediaInfo.vpssGrp);
344 ViStop(&g_aicTennisMediaInfo.viCfg);
345 free(g_aicTennisMediaInfo.viSess);
346 EXIT:
347 SAMPLE_COMM_SYS_Exit();
348 return s32Ret;
349 }