1 /*
2 * Copyright (c) 2022 HiSilicon (Shanghai) Technologies CO., LIMITED.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 /*
17 * 该cpp文件基于OpenCV实现了网球检测功能。为了保证FPS的帧数，
18 * 我们设计的原则是IVE（Intelligent Video Engine）+AI CPU结合使用，即IVE不支持的算子
19 * 通过AI CPU进行计算，否则走IVE硬件加速模块进行处理。并将检测的结果通过VGS标记出来。
20 *
21 * The cpp file implements the tennis ball detection function based on OpenCV.
22 * In order to ensure the number of FPS frames,
23 * The principle of our design is the combination of IVE (Intelligent Video Engine) + AI CPU,
24 * that is, operators not supported by IVE are calculated by AI CPU, otherwise,
25 * the IVE hardware acceleration module is used for processing.
26 * And the detection results are marked by VGS.
27 */
28
29 #include <iostream>
30 #include <opencv2/core.hpp>
31 #include <opencv2/highgui.hpp>
32 #include <opencv2/imgproc.hpp>
33 #include "tennis_detect.h"
34
35 #include "sample_comm_nnie.h"
36 #include "sample_comm_ive.h"
37 #include "sample_media_ai.h"
38 #include "vgs_img.h"
39 #include "misc_util.h"
40
41 using namespace std;
42 using namespace cv;
43
44 static IVE_SRC_IMAGE_S pstSrc;
45 static IVE_DST_IMAGE_S pstDst;
46 static IVE_CSC_CTRL_S stCscCtrl;
47
IveImageParamCfg(IVE_SRC_IMAGE_S * pstSrc,IVE_DST_IMAGE_S * pstDst,VIDEO_FRAME_INFO_S * srcFrame)48 static HI_VOID IveImageParamCfg(IVE_SRC_IMAGE_S *pstSrc, IVE_DST_IMAGE_S *pstDst,
49 VIDEO_FRAME_INFO_S *srcFrame)
50 {
51 pstSrc->enType = IVE_IMAGE_TYPE_YUV420SP;
52 pstSrc->au64VirAddr[0] = srcFrame->stVFrame.u64VirAddr[0];
53 pstSrc->au64VirAddr[1] = srcFrame->stVFrame.u64VirAddr[1];
54 pstSrc->au64VirAddr[2] = srcFrame->stVFrame.u64VirAddr[2]; // 2: Image data virtual address
55
56 pstSrc->au64PhyAddr[0] = srcFrame->stVFrame.u64PhyAddr[0];
57 pstSrc->au64PhyAddr[1] = srcFrame->stVFrame.u64PhyAddr[1];
58 pstSrc->au64PhyAddr[2] = srcFrame->stVFrame.u64PhyAddr[2]; // 2: Image data physical address
59
60 pstSrc->au32Stride[0] = srcFrame->stVFrame.u32Stride[0];
61 pstSrc->au32Stride[1] = srcFrame->stVFrame.u32Stride[1];
62 pstSrc->au32Stride[2] = srcFrame->stVFrame.u32Stride[2]; // 2: Image data span
63
64 pstSrc->u32Width = srcFrame->stVFrame.u32Width;
65 pstSrc->u32Height = srcFrame->stVFrame.u32Height;
66
67 pstDst->enType = IVE_IMAGE_TYPE_U8C3_PACKAGE;
68 pstDst->u32Width = pstSrc->u32Width;
69 pstDst->u32Height = pstSrc->u32Height;
70 pstDst->au32Stride[0] = pstSrc->au32Stride[0];
71 pstDst->au32Stride[1] = 0;
72 pstDst->au32Stride[2] = 0; // 2: Image data span
73 }
74
yuvFrame2rgb(VIDEO_FRAME_INFO_S * srcFrame,IPC_IMAGE * dstImage)75 static HI_S32 yuvFrame2rgb(VIDEO_FRAME_INFO_S *srcFrame, IPC_IMAGE *dstImage)
76 {
77 IVE_HANDLE hIveHandle;
78 HI_S32 s32Ret = 0;
79 stCscCtrl.enMode = IVE_CSC_MODE_PIC_BT709_YUV2RGB; // IVE_CSC_MODE_VIDEO_BT601_YUV2RGB
80 IveImageParamCfg(&pstSrc, &pstDst, srcFrame);
81
82 s32Ret = HI_MPI_SYS_MmzAlloc_Cached(&pstDst.au64PhyAddr[0], (void **)&pstDst.au64VirAddr[0],
83 "User", HI_NULL, pstDst.u32Height*pstDst.au32Stride[0] * 3); // 3: multiple
84 if (HI_SUCCESS != s32Ret) {
85 HI_MPI_SYS_MmzFree(pstDst.au64PhyAddr[0], (void *)pstDst.au64VirAddr[0]);
86 SAMPLE_PRT("HI_MPI_SYS_MmzFree err\n");
87 return s32Ret;
88 }
89
90 s32Ret = HI_MPI_SYS_MmzFlushCache(pstDst.au64PhyAddr[0], (void *)pstDst.au64VirAddr[0],
91 pstDst.u32Height*pstDst.au32Stride[0] * 3); // 3: multiple
92 if (HI_SUCCESS != s32Ret) {
93 HI_MPI_SYS_MmzFree(pstDst.au64PhyAddr[0], (void *)pstDst.au64VirAddr[0]);
94 return s32Ret;
95 }
96 // 3: multiple
97 memset_s((void *)pstDst.au64VirAddr[0], pstDst.u32Height*pstDst.au32Stride[0] * 3,
98 0, pstDst.u32Height*pstDst.au32Stride[0] * 3); // 3: multiple
99 HI_BOOL bInstant = HI_TRUE;
100
101 s32Ret = HI_MPI_IVE_CSC(&hIveHandle, &pstSrc, &pstDst, &stCscCtrl, bInstant);
102 if (HI_SUCCESS != s32Ret) {
103 HI_MPI_SYS_MmzFree(pstDst.au64PhyAddr[0], (void *)pstDst.au64VirAddr[0]);
104 return s32Ret;
105 }
106
107 if (HI_TRUE == bInstant) {
108 HI_BOOL bFinish = HI_TRUE;
109 HI_BOOL bBlock = HI_TRUE;
110 s32Ret = HI_MPI_IVE_Query(hIveHandle, &bFinish, bBlock);
111 while (HI_ERR_IVE_QUERY_TIMEOUT == s32Ret) {
112 usleep(100); // 100: usleep time
113 s32Ret = HI_MPI_IVE_Query(hIveHandle, &bFinish, bBlock);
114 }
115 }
116 dstImage->u64PhyAddr = pstDst.au64PhyAddr[0];
117 dstImage->u64VirAddr = pstDst.au64VirAddr[0];
118 dstImage->u32Width = pstDst.u32Width;
119 dstImage->u32Height = pstDst.u32Height;
120
121 return HI_SUCCESS;
122 }
123
frame2Mat(VIDEO_FRAME_INFO_S * srcFrame,Mat & dstMat)124 static HI_S32 frame2Mat(VIDEO_FRAME_INFO_S *srcFrame, Mat &dstMat)
125 {
126 HI_U32 w = srcFrame->stVFrame.u32Width;
127 HI_U32 h = srcFrame->stVFrame.u32Height;
128 int bufLen = w * h * 3;
129 HI_U8 *srcRGB = NULL;
130 IPC_IMAGE dstImage;
131 if (yuvFrame2rgb(srcFrame, &dstImage) != HI_SUCCESS) {
132 SAMPLE_PRT("yuvFrame2rgb err\n");
133 return HI_FAILURE;
134 }
135 srcRGB = (HI_U8 *)dstImage.u64VirAddr;
136 dstMat.create(h, w, CV_8UC3);
137 memcpy_s(dstMat.data, bufLen * sizeof(HI_U8), srcRGB, bufLen * sizeof(HI_U8));
138 HI_MPI_SYS_MmzFree(dstImage.u64PhyAddr, (void *)&(dstImage.u64VirAddr));
139 return HI_SUCCESS;
140 }
141
TennisDetectLoad(uintptr_t * model)142 HI_S32 tennis_detect::TennisDetectLoad(uintptr_t* model)
143 {
144 HI_S32 ret = 1;
145 *model = 1;
146 SAMPLE_PRT("TennisDetectLoad success\n");
147
148 return ret;
149 }
150
TennisDetectUnload(uintptr_t model)151 HI_S32 tennis_detect::TennisDetectUnload(uintptr_t model)
152 {
153 model = 0;
154
155 return HI_SUCCESS;
156 }
157
158 /*
159 * 网球检测推理
160 * Tennis detect calculation
161 */
TennisDetectCal(uintptr_t model,VIDEO_FRAME_INFO_S * srcFrm,VIDEO_FRAME_INFO_S * dstFrm)162 HI_S32 tennis_detect::TennisDetectCal(uintptr_t model, VIDEO_FRAME_INFO_S *srcFrm, VIDEO_FRAME_INFO_S *dstFrm)
163 {
164 (void)model;
165 int ret = 0;
166 RectBox boxs[32] = {0}; // 32: TENNIS_OBJ_MAX
167 int j = 0;
168
169 Mat image;
170 frame2Mat(srcFrm, image);
171 if (image.size == 0) {
172 SAMPLE_PRT("image is null\n");
173 return HI_FAILURE;
174 }
175
176 Mat src = image;
177 Mat src1 = src.clone();
178 Mat dst, edge, gray, hsv;
179
180 dst.create(src1.size(), src1.type()); // Create a matrix of the same type and size as src (dst)
181
182 /*
183 * cvtColor运算符用于将图像从一个颜色空间转换到另一个颜色空间
184 * The cvtColor operator is used to convert an image from one color space to another color space
185 */
186 cvtColor(src1, hsv, COLOR_BGR2HSV); // Convert original image to HSV image
187
188 /*
189 * 二值化hsv图像，这里是对绿色背景进行二值化，
190 * 这个参数可以根据需要调整
191 *
192 * Binarize the hsv image, here is to binarize the green background,
193 * this parameter can be adjusted according to requirements
194 */
195 inRange(hsv, Scalar(31, 82, 68), Scalar(65, 248, 255), gray); // 31: B, 82: G, 68:R / 65: B, 248:G, 255:R
196
197 /*
198 * 使用canny算子进行边缘检测
199 * Use canny operator for edge detection
200 */
201 Canny(gray, gray, 3, 9, 3); // 3: threshold1, 9: threshold2, 3: apertureSize
202 vector<vector<Point>> contours;
203 findContours(gray, contours, RETR_EXTERNAL, CHAIN_APPROX_SIMPLE, Point());
204 SAMPLE_PRT("contours.size():%d\n", contours.size());
205
206 for (int i = 0; i < (int)contours.size(); i++) {
207 if (contours.size() > 40) { // 40: contours.size() extremes
208 continue;
209 }
210
211 Rect ret1 = boundingRect(Mat(contours[i]));
212 ret1.x -= 5; // 5: x coordinate translation
213 ret1.y -= 5; // 5: y coordinate translation
214 ret1.width += 10; // 10: Rectangle width plus 10
215 ret1.height += 10; // 10: Rectangle height plus 10
216
217 // 20: Rectangle width and height pixel extremes
218 if ((ret1.width > 20) && (ret1.height > 20)) {
219 boxs[j].xmin = ret1.x * 3; // 3: optimized value
220 boxs[j].ymin = (int)(ret1.y * 2.25); // 2.25: optimized value
221 boxs[j].xmax = boxs[j].xmin + ret1.width * 3; // 3: optimized value
222 boxs[j].ymax = boxs[j].ymin + (int)ret1.height * 2.25; // 2.25: optimized value
223 j++;
224 }
225 }
226 // 25: detect boxesNum
227 if (j > 0 && j <= 25) {
228 SAMPLE_PRT("box num:%d\n", j);
229 MppFrmDrawRects(dstFrm, boxs, j, RGB888_RED, 2); // 2: DRAW_RETC_THICK
230 }
231
232 return ret;
233 }