1 /*
2 * Copyright (c) 2022 HiSilicon (Shanghai) Technologies CO., LIMITED.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include <stdlib.h>
17 #include <string.h>
18 #include <stdio.h>
19 #include <errno.h>
20 #include <sys/prctl.h>
21
22 #include "sample_comm_nnie.h"
23 #include "sample_media_ai.h"
24 #include "ai_infer_process.h"
25 #include "vgs_img.h"
26 #include "ive_img.h"
27 #include "posix_help.h"
28 #include "audio_aac_adp.h"
29 #include "base_interface.h"
30 #include "osd_img.h"
31 #include "cnn_trash_classify.h"
32
33 #ifdef __cplusplus
34 #if __cplusplus
35 extern "C" {
36 #endif
37 #endif /* End of #ifdef __cplusplus */
38
39 #define MODEL_FILE_TRASH "/userdata/models/cnn_trash_classify/resnet_inst.wk" // Open source model conversion
40 #define SCORE_MAX 4096 // The score corresponding to the maximum probability
41 #define DETECT_OBJ_MAX 32
42 #define RET_NUM_MAX 4
43 #define THRESH_MIN 30 // Acceptable probability threshold (over this value will be returned to the app)
44
45 #define FRM_WIDTH 256
46 #define FRM_HEIGHT 256
47 #define TXT_BEGX 20
48 #define TXT_BEGY 20
49
50 static int g_num = 108;
51 static int g_count = 0;
52 #define AUDIO_CASE_TWO 2
53 #define AUDIO_SCORE 40 // Confidence can be configured by yourself
54 #define AUDIO_FRAME 14 // Recognize once every 15 frames, can be configured by yourself
55
56 #define MULTIPLE_OF_EXPANSION 100 // Multiple of expansion
57 #define UNKOWN_WASTE 20 // Unkown Waste
58 #define BUFFER_SIZE 16 // buffer size
59 #define MIN_OF_BOX 16 // min of box
60 #define MAX_OF_BOX 240 // max of box
61
62 static HI_BOOL g_bAudioProcessStopSignal = HI_FALSE;
63 static pthread_t g_audioProcessThread = 0;
64 static OsdSet* g_osdsTrash = NULL;
65 static HI_S32 g_osd0Trash = -1;
66
67 static SkPair g_stmChn = {
68 .in = -1,
69 .out = -1
70 };
71
PlayAudio(const RecogNumInfo items)72 static HI_VOID PlayAudio(const RecogNumInfo items)
73 {
74 if (g_count < AUDIO_FRAME) {
75 g_count++;
76 return;
77 }
78
79 const RecogNumInfo *item = &items;
80 uint32_t score = item->score * MULTIPLE_OF_EXPANSION / SCORE_MAX;
81 if ((score > AUDIO_SCORE) && (g_num != item->num)) {
82 g_num = item->num;
83 if (g_num != UNKOWN_WASTE) {
84 AudioTest(g_num, -1);
85 }
86 }
87 g_count = 0;
88 }
89
GetAudioFileName(HI_VOID * arg)90 static HI_VOID* GetAudioFileName(HI_VOID* arg)
91 {
92 RecogNumInfo resBuf = {0};
93 int ret;
94
95 while (g_bAudioProcessStopSignal == false) {
96 ret = FdReadMsg(g_stmChn.in, &resBuf, sizeof(RecogNumInfo));
97 if (ret == sizeof(RecogNumInfo)) {
98 PlayAudio(resBuf);
99 }
100 }
101
102 return NULL;
103 }
104
CnnTrashClassifyLoadModel(uintptr_t * model,OsdSet * osds)105 HI_S32 CnnTrashClassifyLoadModel(uintptr_t* model, OsdSet* osds)
106 {
107 SAMPLE_SVP_NNIE_CFG_S *self = NULL;
108 HI_S32 ret;
109 HI_CHAR audioThreadName[BUFFER_SIZE] = {0};
110
111 ret = OsdLibInit();
112 HI_ASSERT(ret == HI_SUCCESS);
113
114 g_osdsTrash = osds;
115 HI_ASSERT(g_osdsTrash);
116 g_osd0Trash = OsdsCreateRgn(g_osdsTrash);
117 HI_ASSERT(g_osd0Trash >= 0);
118
119 ret = CnnCreate(&self, MODEL_FILE_TRASH);
120 *model = ret < 0 ? 0 : (uintptr_t)self;
121 SAMPLE_PRT("load cnn trash classify model, ret:%d\n", ret);
122
123 if (GetCfgBool("audio_player:support_audio", true)) {
124 ret = SkPairCreate(&g_stmChn);
125 HI_ASSERT(ret == 0);
126 if (snprintf_s(audioThreadName, BUFFER_SIZE, BUFFER_SIZE - 1, "AudioProcess") < 0) {
127 HI_ASSERT(0);
128 }
129 prctl(PR_SET_NAME, (unsigned long)audioThreadName, 0, 0, 0);
130 ret = pthread_create(&g_audioProcessThread, NULL, GetAudioFileName, NULL);
131 if (ret != 0) {
132 SAMPLE_PRT("audio proccess thread creat fail:%s\n", strerror(ret));
133 return ret;
134 }
135 }
136
137 return ret;
138 }
139
CnnTrashClassifyUnloadModel(uintptr_t model)140 HI_S32 CnnTrashClassifyUnloadModel(uintptr_t model)
141 {
142 CnnDestroy((SAMPLE_SVP_NNIE_CFG_S*)model);
143 SAMPLE_PRT("unload trash classify model success\n");
144 OsdsClear(g_osdsTrash);
145
146 if (GetCfgBool("audio_player:support_audio", true)) {
147 SkPairDestroy(&g_stmChn);
148 SAMPLE_PRT("SkPairDestroy success\n");
149 g_bAudioProcessStopSignal = HI_TRUE;
150 pthread_join(g_audioProcessThread, NULL);
151 g_audioProcessThread = 0;
152 }
153
154 return HI_SUCCESS;
155 }
156
CnnTrashClassifyFlag(const RecogNumInfo items[],HI_S32 itemNum,HI_CHAR * buf,HI_S32 size)157 static HI_S32 CnnTrashClassifyFlag(const RecogNumInfo items[], HI_S32 itemNum, HI_CHAR* buf, HI_S32 size)
158 {
159 HI_S32 offset = 0;
160 HI_CHAR *trashName = NULL;
161
162 offset += snprintf_s(buf + offset, size - offset, size - offset - 1, "trash classify: {");
163 for (HI_U32 i = 0; i < itemNum; i++) {
164 const RecogNumInfo *item = &items[i];
165 uint32_t score = item->score * HI_PER_BASE / SCORE_MAX;
166 if (score < THRESH_MIN) {
167 break;
168 }
169 SAMPLE_PRT("----trash item flag----num:%d, score:%d\n", item->num, score);
170 switch (item->num) {
171 case 0u:
172 case 1u:
173 case 2u:
174 case 3u:
175 case 4u:
176 case 5u:
177 trashName = "Kitchen Waste";
178 break;
179 case 6u:
180 case 7u:
181 case 8u:
182 case 9u:
183 trashName = "Harmful Waste";
184 break;
185 case 10u:
186 case 11u:
187 case 12u:
188 case 13u:
189 case 14u:
190 case 15u:
191 trashName = "Recyle Waste";
192 break;
193 case 16u:
194 case 17u:
195 case 18u:
196 case 19u:
197 trashName = "Other Waste";
198 break;
199 default:
200 trashName = "Unkown Waste";
201 break;
202 }
203 offset += snprintf_s(buf + offset, size - offset, size - offset - 1,
204 "%s%s %u:%u%%", (i == 0 ? " " : ", "), trashName, (int)item->num, (int)score);
205 HI_ASSERT(offset < size);
206 }
207 offset += snprintf_s(buf + offset, size - offset, size - offset - 1, " }");
208 HI_ASSERT(offset < size);
209 return HI_SUCCESS;
210 }
211
CnnTrashClassifyCal(uintptr_t model,VIDEO_FRAME_INFO_S * srcFrm,VIDEO_FRAME_INFO_S * resFrm)212 HI_S32 CnnTrashClassifyCal(uintptr_t model, VIDEO_FRAME_INFO_S *srcFrm, VIDEO_FRAME_INFO_S *resFrm)
213 {
214 SAMPLE_PRT("begin CnnTrashClassifyCal\n");
215 SAMPLE_SVP_NNIE_CFG_S *self = (SAMPLE_SVP_NNIE_CFG_S*)model; // reference to SDK sample_comm_nnie.h Line 99
216 IVE_IMAGE_S img; // referece to SDK hi_comm_ive.h Line 143
217 RectBox cnnBoxs[DETECT_OBJ_MAX] = {0};
218 VIDEO_FRAME_INFO_S resizeFrm; // Meet the input frame of the plug
219 static HI_CHAR prevOsd[NORM_BUF_SIZE] = "";
220 HI_CHAR osdBuf[NORM_BUF_SIZE] = "";
221 /*
222 01-Kitchen_Watermelon_rind 02_Kitchen_Egg_shell
223 03_Kitchen_Fishbone 04_Kitchen_Eggplant
224 05_Kitchen_Scallion 06_Kitchen_Mushromm
225 07_Hazardous_Waste_battery 08_Hazardous_Expired_cosmetrics
226 09_Hazardous_Woundplast 10_Hazardous_Medical_gauze
227 11_Recyclabel_Old_dolls 12_Recyclabel_Old_clip
228 13_Recyclabel_Toothbrush 14_Recyclabel_Milk_box
229 15_Recyclabel_Old_handbag 16_Recyclabel_Zip_top_can
230 17_other_Ciggrate_end 18_Other_Bad_closestool
231 19_other_Brick 20_Other_Dish
232 21_unkown_waste_or_background
233 */
234 RecogNumInfo resBuf[RET_NUM_MAX] = {0};
235 HI_S32 resLen = 0;
236 HI_S32 ret;
237 IVE_IMAGE_S imgIn;
238
239 cnnBoxs[0].xmin = MIN_OF_BOX;
240 cnnBoxs[0].xmax = MAX_OF_BOX;
241 cnnBoxs[0].ymin = MIN_OF_BOX;
242 cnnBoxs[0].ymax = MAX_OF_BOX;
243
244 ret = MppFrmResize(srcFrm, &resizeFrm, FRM_WIDTH, FRM_HEIGHT); // resize 256*256
245 SAMPLE_CHECK_EXPR_RET(ret != HI_SUCCESS, ret, "for resize FAIL, ret=%x\n", ret);
246
247 ret = FrmToOrigImg(&resizeFrm, &img);
248 SAMPLE_CHECK_EXPR_RET(ret != HI_SUCCESS, ret, "for Frm2Img FAIL, ret=%x\n", ret);
249
250 ret = ImgYuvCrop(&img, &imgIn, &cnnBoxs[0]); // Crop the image to classfication network
251 SAMPLE_CHECK_EXPR_RET(ret < 0, ret, "ImgYuvCrop FAIL, ret=%x\n", ret);
252
253 // Follow the reasoning logic
254 ret = CnnCalU8c1Img(self, &imgIn, resBuf, sizeof(resBuf) / sizeof((resBuf)[0]), &resLen);
255 SAMPLE_CHECK_EXPR_RET(ret < 0, ret, "cnn cal FAIL, ret=%x\n", ret);
256
257 HI_ASSERT(resLen <= sizeof(resBuf) / sizeof(resBuf[0]));
258 ret = CnnTrashClassifyFlag(resBuf, resLen, osdBuf, sizeof(osdBuf));
259 SAMPLE_CHECK_EXPR_RET(ret < 0, ret, "CnnTrashClassifyFlag cal FAIL, ret=%x\n", ret);
260
261 if (GetCfgBool("audio_player:support_audio", true)) {
262 if (FdWriteMsg(g_stmChn.out, &resBuf[0], sizeof(RecogNumInfo)) != sizeof(RecogNumInfo)) {
263 SAMPLE_PRT("FdWriteMsg FAIL\n");
264 }
265 }
266
267 if (strcmp(osdBuf, prevOsd) != 0) {
268 HiStrxfrm(prevOsd, osdBuf, sizeof(prevOsd));
269
270 // Superimpose graphics into resFrm
271 HI_OSD_ATTR_S rgn;
272 TxtRgnInit(&rgn, osdBuf, TXT_BEGX, TXT_BEGY, ARGB1555_YELLOW2); // font width and heigt use default 40
273 OsdsSetRgn(g_osdsTrash, g_osd0Trash, &rgn);
274 ret = HI_MPI_VPSS_SendFrame(0, 0, srcFrm, 0);
275 if (ret != HI_SUCCESS) {
276 SAMPLE_PRT("Error(%#x), HI_MPI_VPSS_SendFrame failed!\n", ret);
277 }
278 }
279
280 IveImgDestroy(&imgIn);
281 MppFrmDestroy(&resizeFrm);
282
283 return ret;
284 }
285
286 #ifdef __cplusplus
287 #if __cplusplus
288 }
289 #endif
290 #endif /* End of #ifdef __cplusplus */
291