1 /*
2 * Copyright (c) 2022 HiSilicon (Shanghai) Technologies CO., LIMITED.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 /*
17 * 本文件将垃圾分类wk模型部署到板端，通过NNIE硬件加速进行推理。该文件提供了垃圾分类场景的API接口，
18 * 包括模型的加载、模型的卸载、模型的推理、AI flag业务处理接口。支持语音实时播放功能。
19 *
20 * This file deploys the trash classification wk model to the board,
21 * and performs inference through NNIE hardware acceleration.
22 * This file provides API interfaces for trash classification scenarios,
23 * including model loading, model unloading, model reasoning,
24 * and AI flag business processing interfaces. Support audio real-time playback function.
25 */
26
27 #include <stdlib.h>
28 #include <string.h>
29 #include <stdio.h>
30 #include <errno.h>
31 #include <sys/prctl.h>
32
33 #include "sample_comm_nnie.h"
34 #include "sample_media_ai.h"
35 #include "ai_infer_process.h"
36 #include "vgs_img.h"
37 #include "ive_img.h"
38 #include "posix_help.h"
39 #include "audio_aac_adp.h"
40 #include "base_interface.h"
41 #include "osd_img.h"
42 #include "cnn_trash_classify.h"
43
44 #ifdef __cplusplus
45 #if __cplusplus
46 extern "C" {
47 #endif
48 #endif /* End of #ifdef __cplusplus */
49
50 #define MODEL_FILE_TRASH "/userdata/models/cnn_trash_classify/resnet_inst.wk" // Open source model conversion
51 #define SCORE_MAX 4096 // The score corresponding to the maximum probability
52 #define DETECT_OBJ_MAX 32
53 #define RET_NUM_MAX 4
54 #define THRESH_MIN 30 // Acceptable probability threshold (over this value will be returned to the app)
55
56 #define FRM_WIDTH 256
57 #define FRM_HEIGHT 256
58 #define TXT_BEGX 20
59 #define TXT_BEGY 20
60
61 static int g_num = 108;
62 static int g_count = 0;
63 #define AUDIO_CASE_TWO 2
64 #define AUDIO_SCORE 40 // Confidence can be configured by yourself
65 #define AUDIO_FRAME 14 // Recognize once every 15 frames, can be configured by yourself
66
67 #define MULTIPLE_OF_EXPANSION 100 // Multiple of expansion
68 #define UNKOWN_WASTE 20 // Unkown Waste
69 #define BUFFER_SIZE 16 // buffer size
70 #define MIN_OF_BOX 16 // min of box
71 #define MAX_OF_BOX 240 // max of box
72
73 static HI_BOOL g_bAudioProcessStopSignal = HI_FALSE;
74 static pthread_t g_audioProcessThread = 0;
75 static OsdSet* g_osdsTrash = NULL;
76 static HI_S32 g_osd0Trash = -1;
77
78 static SkPair g_stmChn = {
79 .in = -1,
80 .out = -1
81 };
82
83 /*
84 * 将识别的结果进行音频播放
85 * Audio playback of the recognition results
86 */
PlayAudio(const RecogNumInfo items)87 static HI_VOID PlayAudio(const RecogNumInfo items)
88 {
89 if (g_count < AUDIO_FRAME) {
90 g_count++;
91 return;
92 }
93
94 const RecogNumInfo *item = &items;
95 uint32_t score = item->score * MULTIPLE_OF_EXPANSION / SCORE_MAX;
96 if ((score > AUDIO_SCORE) && (g_num != item->num)) {
97 g_num = item->num;
98 if (g_num != UNKOWN_WASTE) {
99 AudioTest(g_num, -1);
100 }
101 }
102 g_count = 0;
103 }
104
GetAudioFileName(HI_VOID * arg)105 static HI_VOID* GetAudioFileName(HI_VOID* arg)
106 {
107 RecogNumInfo resBuf = {0};
108 int ret;
109
110 while (g_bAudioProcessStopSignal == false) {
111 ret = FdReadMsg(g_stmChn.in, &resBuf, sizeof(RecogNumInfo));
112 if (ret == sizeof(RecogNumInfo)) {
113 PlayAudio(resBuf);
114 }
115 }
116
117 return NULL;
118 }
119
120 /*
121 * 加载垃圾分类wk模型
122 * Load the trash classification wk model
123 */
CnnTrashClassifyLoadModel(uintptr_t * model,OsdSet * osds)124 HI_S32 CnnTrashClassifyLoadModel(uintptr_t* model, OsdSet* osds)
125 {
126 SAMPLE_SVP_NNIE_CFG_S *self = NULL;
127 HI_S32 ret;
128 HI_CHAR audioThreadName[BUFFER_SIZE] = {0};
129
130 ret = OsdLibInit();
131 HI_ASSERT(ret == HI_SUCCESS);
132
133 g_osdsTrash = osds;
134 HI_ASSERT(g_osdsTrash);
135 g_osd0Trash = OsdsCreateRgn(g_osdsTrash);
136 HI_ASSERT(g_osd0Trash >= 0);
137
138 ret = CnnCreate(&self, MODEL_FILE_TRASH);
139 *model = ret < 0 ? 0 : (uintptr_t)self;
140 SAMPLE_PRT("load cnn trash classify model, ret:%d\n", ret);
141
142 if (GetCfgBool("audio_player:support_audio", true)) {
143 ret = SkPairCreate(&g_stmChn);
144 HI_ASSERT(ret == 0);
145 if (snprintf_s(audioThreadName, BUFFER_SIZE, BUFFER_SIZE - 1, "AudioProcess") < 0) {
146 HI_ASSERT(0);
147 }
148 prctl(PR_SET_NAME, (unsigned long)audioThreadName, 0, 0, 0);
149 ret = pthread_create(&g_audioProcessThread, NULL, GetAudioFileName, NULL);
150 if (ret != 0) {
151 SAMPLE_PRT("audio proccess thread creat fail:%s\n", strerror(ret));
152 return ret;
153 }
154 }
155
156 return ret;
157 }
158
159 /*
160 * 卸载垃圾分类wk模型
161 * Unload the trash classification wk model
162 */
CnnTrashClassifyUnloadModel(uintptr_t model)163 HI_S32 CnnTrashClassifyUnloadModel(uintptr_t model)
164 {
165 CnnDestroy((SAMPLE_SVP_NNIE_CFG_S*)model);
166 SAMPLE_PRT("unload trash classify model success\n");
167 OsdsClear(g_osdsTrash);
168
169 if (GetCfgBool("audio_player:support_audio", true)) {
170 SkPairDestroy(&g_stmChn);
171 SAMPLE_PRT("SkPairDestroy success\n");
172 g_bAudioProcessStopSignal = HI_TRUE;
173 pthread_join(g_audioProcessThread, NULL);
174 g_audioProcessThread = 0;
175 }
176
177 return HI_SUCCESS;
178 }
179
180 /*
181 * 根据推理结果进行业务处理
182 * Perform business processing based on inference results
183 */
CnnTrashClassifyFlag(const RecogNumInfo items[],HI_S32 itemNum,HI_CHAR * buf,HI_S32 size)184 static HI_S32 CnnTrashClassifyFlag(const RecogNumInfo items[], HI_S32 itemNum, HI_CHAR* buf, HI_S32 size)
185 {
186 HI_S32 offset = 0;
187 HI_CHAR *trashName = NULL;
188
189 offset += snprintf_s(buf + offset, size - offset, size - offset - 1, "trash classify: {");
190 for (HI_U32 i = 0; i < itemNum; i++) {
191 const RecogNumInfo *item = &items[i];
192 uint32_t score = item->score * HI_PER_BASE / SCORE_MAX;
193 if (score < THRESH_MIN) {
194 break;
195 }
196 SAMPLE_PRT("----trash item flag----num:%d, score:%d\n", item->num, score);
197 switch (item->num) {
198 case 0u:
199 case 1u:
200 case 2u:
201 case 3u:
202 case 4u:
203 case 5u:
204 trashName = "Kitchen Waste";
205 break;
206 case 6u:
207 case 7u:
208 case 8u:
209 case 9u:
210 trashName = "Harmful Waste";
211 break;
212 case 10u:
213 case 11u:
214 case 12u:
215 case 13u:
216 case 14u:
217 case 15u:
218 trashName = "Recyle Waste";
219 break;
220 case 16u:
221 case 17u:
222 case 18u:
223 case 19u:
224 trashName = "Other Waste";
225 break;
226 default:
227 trashName = "Unkown Waste";
228 break;
229 }
230 offset += snprintf_s(buf + offset, size - offset, size - offset - 1,
231 "%s%s %u:%u%%", (i == 0 ? " " : ", "), trashName, (int)item->num, (int)score);
232 HI_ASSERT(offset < size);
233 }
234 offset += snprintf_s(buf + offset, size - offset, size - offset - 1, " }");
235 HI_ASSERT(offset < size);
236 return HI_SUCCESS;
237 }
238
239 /*
240 * 先进行预处理，再使用NNIE进行硬件加速推理，不支持层通过AI CPU进行计算
241 *
242 * Perform preprocessing first, and then use NNIE for hardware accelerated inference,
243 * and do not support layers to be calculated by AI CPU
244 */
CnnTrashClassifyCal(uintptr_t model,VIDEO_FRAME_INFO_S * srcFrm,VIDEO_FRAME_INFO_S * resFrm)245 HI_S32 CnnTrashClassifyCal(uintptr_t model, VIDEO_FRAME_INFO_S *srcFrm, VIDEO_FRAME_INFO_S *resFrm)
246 {
247 SAMPLE_PRT("begin CnnTrashClassifyCal\n");
248 SAMPLE_SVP_NNIE_CFG_S *self = (SAMPLE_SVP_NNIE_CFG_S*)model; // reference to SDK sample_comm_nnie.h Line 99
249 IVE_IMAGE_S img; // referece to SDK hi_comm_ive.h Line 143
250 RectBox cnnBoxs[DETECT_OBJ_MAX] = {0};
251 VIDEO_FRAME_INFO_S resizeFrm; // Meet the input frame of the plug
252 static HI_CHAR prevOsd[NORM_BUF_SIZE] = "";
253 HI_CHAR osdBuf[NORM_BUF_SIZE] = "";
254 /*
255 01-Kitchen_Watermelon_rind 02_Kitchen_Egg_shell
256 03_Kitchen_Fishbone 04_Kitchen_Eggplant
257 05_Kitchen_Scallion 06_Kitchen_Mushromm
258 07_Hazardous_Waste_battery 08_Hazardous_Expired_cosmetrics
259 09_Hazardous_Woundplast 10_Hazardous_Medical_gauze
260 11_Recyclabel_Old_dolls 12_Recyclabel_Old_clip
261 13_Recyclabel_Toothbrush 14_Recyclabel_Milk_box
262 15_Recyclabel_Old_handbag 16_Recyclabel_Zip_top_can
263 17_other_Ciggrate_end 18_Other_Bad_closestool
264 19_other_Brick 20_Other_Dish
265 21_unkown_waste_or_background
266 */
267 RecogNumInfo resBuf[RET_NUM_MAX] = {0};
268 HI_S32 resLen = 0;
269 HI_S32 ret;
270 IVE_IMAGE_S imgIn;
271
272 cnnBoxs[0].xmin = MIN_OF_BOX;
273 cnnBoxs[0].xmax = MAX_OF_BOX;
274 cnnBoxs[0].ymin = MIN_OF_BOX;
275 cnnBoxs[0].ymax = MAX_OF_BOX;
276
277 ret = MppFrmResize(srcFrm, &resizeFrm, FRM_WIDTH, FRM_HEIGHT); // resize 256*256
278 SAMPLE_CHECK_EXPR_RET(ret != HI_SUCCESS, ret, "for resize FAIL, ret=%x\n", ret);
279
280 ret = FrmToOrigImg(&resizeFrm, &img);
281 SAMPLE_CHECK_EXPR_RET(ret != HI_SUCCESS, ret, "for Frm2Img FAIL, ret=%x\n", ret);
282
283 ret = ImgYuvCrop(&img, &imgIn, &cnnBoxs[0]); // Crop the image to classfication network
284 SAMPLE_CHECK_EXPR_RET(ret < 0, ret, "ImgYuvCrop FAIL, ret=%x\n", ret);
285
286 ret = CnnCalImg(self, &imgIn, resBuf, sizeof(resBuf) / sizeof((resBuf)[0]), &resLen);
287 SAMPLE_CHECK_EXPR_RET(ret < 0, ret, "cnn cal FAIL, ret=%x\n", ret);
288
289 HI_ASSERT(resLen <= sizeof(resBuf) / sizeof(resBuf[0]));
290 ret = CnnTrashClassifyFlag(resBuf, resLen, osdBuf, sizeof(osdBuf));
291 SAMPLE_CHECK_EXPR_RET(ret < 0, ret, "CnnTrashClassifyFlag cal FAIL, ret=%x\n", ret);
292
293 if (GetCfgBool("audio_player:support_audio", true)) {
294 if (FdWriteMsg(g_stmChn.out, &resBuf[0], sizeof(RecogNumInfo)) != sizeof(RecogNumInfo)) {
295 SAMPLE_PRT("FdWriteMsg FAIL\n");
296 }
297 }
298
299 /*
300 * 仅当计算结果与之前计算发生变化时，才重新打OSD输出文字
301 * Only when the calculation result changes from the previous calculation, re-print the OSD output text
302 */
303 if (strcmp(osdBuf, prevOsd) != 0) {
304 HiStrxfrm(prevOsd, osdBuf, sizeof(prevOsd));
305 HI_OSD_ATTR_S rgn;
306 TxtRgnInit(&rgn, osdBuf, TXT_BEGX, TXT_BEGY, ARGB1555_YELLOW2); // font width and heigt use default 40
307 OsdsSetRgn(g_osdsTrash, g_osd0Trash, &rgn);
308 /*
309 * 用户向VPSS发送数据
310 * User sends data to VPSS
311 */
312 ret = HI_MPI_VPSS_SendFrame(0, 0, srcFrm, 0);
313 if (ret != HI_SUCCESS) {
314 SAMPLE_PRT("Error(%#x), HI_MPI_VPSS_SendFrame failed!\n", ret);
315 }
316 }
317
318 IveImgDestroy(&imgIn);
319 MppFrmDestroy(&resizeFrm);
320
321 return ret;
322 }
323
324 #ifdef __cplusplus
325 #if __cplusplus
326 }
327 #endif
328 #endif /* End of #ifdef __cplusplus */
329