1 /*
2 * Copyright (c) 2022 HiSilicon (Shanghai) Technologies CO., LIMITED.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include <stdlib.h>
17 #include <string.h>
18 #include <stdio.h>
19 #include <errno.h>
20
21 #include "sample_comm_nnie.h"
22 #include "sample_media_ai.h"
23 #include "ai_infer_process.h"
24 #include "yolov2_hand_detect.h"
25 #include "vgs_img.h"
26 #include "ive_img.h"
27 #include "misc_util.h"
28 #include "hisignalling.h"
29
30 #ifdef __cplusplus
31 #if __cplusplus
32 extern "C" {
33 #endif
34 #endif /* End of #ifdef __cplusplus */
35
36 #define HAND_FRM_WIDTH 640
37 #define HAND_FRM_HEIGHT 384
38 #define DETECT_OBJ_MAX 32
39 #define RET_NUM_MAX 4
40 #define DRAW_RETC_THICK 2 // Draw the width of the line
41 #define WIDTH_LIMIT 32
42 #define HEIGHT_LIMIT 32
43 #define IMAGE_WIDTH 224 // The resolution of the model IMAGE sent to the classification is 224*224
44 #define IMAGE_HEIGHT 224
45 #define MODEL_FILE_GESTURE "/userdata/models/hand_classify/hand_gesture.wk" // darknet framework wk model
46
47 static int biggestBoxIndex;
48 static IVE_IMAGE_S img;
49 static DetectObjInfo objs[DETECT_OBJ_MAX] = {0};
50 static RectBox boxs[DETECT_OBJ_MAX] = {0};
51 static RectBox objBoxs[DETECT_OBJ_MAX] = {0};
52 static RectBox remainingBoxs[DETECT_OBJ_MAX] = {0};
53 static RectBox cnnBoxs[DETECT_OBJ_MAX] = {0}; // Store the results of the classification network
54 static RecogNumInfo numInfo[RET_NUM_MAX] = {0};
55 static IVE_IMAGE_S imgIn;
56 static IVE_IMAGE_S imgDst;
57 static VIDEO_FRAME_INFO_S frmIn;
58 static VIDEO_FRAME_INFO_S frmDst;
59 int uartFd = 0;
Yolo2HandDetectResnetClassifyLoad(uintptr_t * model)60 HI_S32 Yolo2HandDetectResnetClassifyLoad(uintptr_t* model)
61 {
62 SAMPLE_SVP_NNIE_CFG_S *self = NULL;
63 HI_S32 ret;
64
65 ret = CnnCreate(&self, MODEL_FILE_GESTURE);
66 *model = ret < 0 ? 0 : (uintptr_t)self;
67 HandDetectInit(); // Initialize the hand detection model
68 SAMPLE_PRT("Load hand detect claasify model success\n");
69 /* uart open init */
70 uartFd = UartOpenInit();
71 if (uartFd < 0) {
72 printf("uart1 open failed\r\n");
73 } else {
74 printf("uart1 open successed\r\n");
75 }
76 return ret;
77 }
78
Yolo2HandDetectResnetClassifyUnload(uintptr_t model)79 HI_S32 Yolo2HandDetectResnetClassifyUnload(uintptr_t model)
80 {
81 CnnDestroy((SAMPLE_SVP_NNIE_CFG_S*)model);
82 HandDetectExit(); // Uninitialize the hand detection model
83 SAMPLE_PRT("Unload hand detect claasify model success\n");
84
85 return 0;
86 }
87
88 /* Get the maximum hand */
GetBiggestHandIndex(RectBox boxs[],int detectNum)89 static HI_S32 GetBiggestHandIndex(RectBox boxs[], int detectNum)
90 {
91 HI_S32 handIndex = 0;
92 HI_S32 biggestBoxIndex = handIndex;
93 HI_S32 biggestBoxWidth = boxs[handIndex].xmax - boxs[handIndex].xmin + 1;
94 HI_S32 biggestBoxHeight = boxs[handIndex].ymax - boxs[handIndex].ymin + 1;
95 HI_S32 biggestBoxArea = biggestBoxWidth * biggestBoxHeight;
96
97 for (handIndex = 1; handIndex < detectNum; handIndex++) {
98 HI_S32 boxWidth = boxs[handIndex].xmax - boxs[handIndex].xmin + 1;
99 HI_S32 boxHeight = boxs[handIndex].ymax - boxs[handIndex].ymin + 1;
100 HI_S32 boxArea = boxWidth * boxHeight;
101 if (biggestBoxArea < boxArea) {
102 biggestBoxArea = boxArea;
103 biggestBoxIndex = handIndex;
104 }
105 biggestBoxWidth = boxs[biggestBoxIndex].xmax - boxs[biggestBoxIndex].xmin + 1;
106 biggestBoxHeight = boxs[biggestBoxIndex].ymax - boxs[biggestBoxIndex].ymin + 1;
107 }
108
109 if ((biggestBoxWidth == 1) || (biggestBoxHeight == 1) || (detectNum == 0)) {
110 biggestBoxIndex = -1;
111 }
112
113 return biggestBoxIndex;
114 }
115
116 /* hand gesture recognition info */
HandDetectFlag(const RecogNumInfo resBuf)117 static void HandDetectFlag(const RecogNumInfo resBuf)
118 {
119 HI_CHAR *gestureName = NULL;
120 switch (resBuf.num) {
121 case 0u:
122 gestureName = "gesture fist";
123 UartSendRead(uartFd, FistGesture); // 拳头手势
124 SAMPLE_PRT("----gesture name----:%s\n", gestureName);
125 break;
126 case 1u:
127 gestureName = "gesture indexUp";
128 UartSendRead(uartFd, ForefingerGesture); // 食指手势
129 SAMPLE_PRT("----gesture name----:%s\n", gestureName);
130 break;
131 case 2u:
132 gestureName = "gesture OK";
133 UartSendRead(uartFd, OkGesture); // OK手势
134 SAMPLE_PRT("----gesture name----:%s\n", gestureName);
135 break;
136 case 3u:
137 gestureName = "gesture palm";
138 UartSendRead(uartFd, PalmGesture); // 手掌手势
139 SAMPLE_PRT("----gesture name----:%s\n", gestureName);
140 break;
141 case 4u:
142 gestureName = "gesture yes";
143 UartSendRead(uartFd, YesGesture); // yes手势
144 SAMPLE_PRT("----gesture name----:%s\n", gestureName);
145 break;
146 case 5u:
147 gestureName = "gesture pinchOpen";
148 UartSendRead(uartFd, ForefingerAndThumbGesture); // 食指 + 大拇指
149 SAMPLE_PRT("----gesture name----:%s\n", gestureName);
150 break;
151 case 6u:
152 gestureName = "gesture phoneCall";
153 UartSendRead(uartFd, LittleFingerAndThumbGesture); // 大拇指 + 小拇指
154 SAMPLE_PRT("----gesture name----:%s\n", gestureName);
155 break;
156 default:
157 gestureName = "gesture others";
158 UartSendRead(uartFd, InvalidGesture); // 无效值
159 SAMPLE_PRT("----gesture name----:%s\n", gestureName);
160 break;
161 }
162 SAMPLE_PRT("hand gesture success\n");
163 }
164
Yolo2HandDetectResnetClassifyCal(uintptr_t model,VIDEO_FRAME_INFO_S * srcFrm,VIDEO_FRAME_INFO_S * dstFrm)165 HI_S32 Yolo2HandDetectResnetClassifyCal(uintptr_t model, VIDEO_FRAME_INFO_S *srcFrm, VIDEO_FRAME_INFO_S *dstFrm)
166 {
167 SAMPLE_SVP_NNIE_CFG_S *self = (SAMPLE_SVP_NNIE_CFG_S*)model;
168 HI_S32 resLen = 0;
169 int objNum;
170 int ret;
171 int num = 0;
172
173 ret = FrmToOrigImg((VIDEO_FRAME_INFO_S*)srcFrm, &img);
174 SAMPLE_CHECK_EXPR_RET(ret != HI_SUCCESS, ret, "hand detect for YUV Frm to Img FAIL, ret=%#x\n", ret);
175
176 objNum = HandDetectCal(&img, objs); // Send IMG to the detection net for reasoning
177 for (int i = 0; i < objNum; i++) {
178 cnnBoxs[i] = objs[i].box;
179 RectBox *box = &objs[i].box;
180 RectBoxTran(box, HAND_FRM_WIDTH, HAND_FRM_HEIGHT,
181 dstFrm->stVFrame.u32Width, dstFrm->stVFrame.u32Height);
182 SAMPLE_PRT("yolo2_out: {%d, %d, %d, %d}\n", box->xmin, box->ymin, box->xmax, box->ymax);
183 boxs[i] = *box;
184 }
185 biggestBoxIndex = GetBiggestHandIndex(boxs, objNum);
186 SAMPLE_PRT("biggestBoxIndex:%d, objNum:%d\n", biggestBoxIndex, objNum);
187
188 // When an object is detected, a rectangle is drawn in the DSTFRM
189 if (biggestBoxIndex >= 0) {
190 objBoxs[0] = boxs[biggestBoxIndex];
191 MppFrmDrawRects(dstFrm, objBoxs, 1, RGB888_GREEN, DRAW_RETC_THICK); // Target hand objnum is equal to 1
192
193 for (int j = 0; (j < objNum) && (objNum > 1); j++) {
194 if (j != biggestBoxIndex) {
195 remainingBoxs[num++] = boxs[j];
196 // others hand objnum is equal to objnum -1
197 MppFrmDrawRects(dstFrm, remainingBoxs, objNum - 1, RGB888_RED, DRAW_RETC_THICK);
198 }
199 }
200
201 // Crop the image to classification network
202 ret = ImgYuvCrop(&img, &imgIn, &cnnBoxs[biggestBoxIndex]);
203 SAMPLE_CHECK_EXPR_RET(ret < 0, ret, "ImgYuvCrop FAIL, ret=%#x\n", ret);
204
205 if ((imgIn.u32Width >= WIDTH_LIMIT) && (imgIn.u32Height >= HEIGHT_LIMIT)) {
206 COMPRESS_MODE_E enCompressMode = srcFrm->stVFrame.enCompressMode;
207 ret = OrigImgToFrm(&imgIn, &frmIn);
208 frmIn.stVFrame.enCompressMode = enCompressMode;
209 SAMPLE_PRT("crop u32Width = %d, img.u32Height = %d\n", imgIn.u32Width, imgIn.u32Height);
210 ret = MppFrmResize(&frmIn, &frmDst, IMAGE_WIDTH, IMAGE_HEIGHT);
211 ret = FrmToOrigImg(&frmDst, &imgDst);
212 ret = CnnCalU8c1Img(self, &imgDst, numInfo, sizeof(numInfo) / sizeof((numInfo)[0]), &resLen);
213 SAMPLE_CHECK_EXPR_RET(ret < 0, ret, "CnnCalU8c1Img FAIL, ret=%#x\n", ret);
214 HI_ASSERT(resLen <= sizeof(numInfo) / sizeof(numInfo[0]));
215 HandDetectFlag(numInfo[0]);
216 MppFrmDestroy(&frmDst);
217 }
218 IveImgDestroy(&imgIn);
219 }
220
221 return ret;
222 }
223
224 #ifdef __cplusplus
225 #if __cplusplus
226 }
227 #endif
228 #endif /* End of #ifdef __cplusplus */
229