1 /**
2 * Copyright 2023 Huawei Technologies Co., Ltd
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "nnacl/kernel/crop_and_resize.h"
18 #include "nnacl/kernel/default_kernel_base.h"
19 #include "nnacl/fp32/resize_fp32.h"
20 #include "nnacl/tensor_c_utils.h"
21
CropAndResizeMallocTmpBuffer(CropAndResizeStruct * crop_and_resize)22 int CropAndResizeMallocTmpBuffer(CropAndResizeStruct *crop_and_resize) {
23 TensorC *input_tensor = crop_and_resize->base_.in_[FIRST_INPUT];
24 NNACL_CHECK_NULL_RETURN_ERR(input_tensor);
25 TensorC *output_tensor = crop_and_resize->base_.out_[OUTPUT_INDEX];
26 NNACL_CHECK_NULL_RETURN_ERR(output_tensor);
27 ExecEnv *env = crop_and_resize->base_.env_;
28 NNACL_CHECK_NULL_RETURN_ERR(env);
29
30 // Malloc buffer to save coordinate.
31 // For mode CROP_AND_RESIZE, different output batches require different cache coordinates.
32 crop_and_resize->batch_ = GetBatch(output_tensor);
33 NNACL_CHECK_INT_MUL_NOT_OVERFLOW(crop_and_resize->new_height_, crop_and_resize->batch_, NNACL_ERR);
34 int height_size = crop_and_resize->new_height_ * crop_and_resize->batch_;
35 NNACL_CHECK_MALLOC_SIZE(height_size);
36 crop_and_resize->y_bottoms_ = (int *)env->Alloc(env->allocator_, height_size * sizeof(int));
37 NNACL_MALLOC_CHECK_NULL_RETURN_ERR(crop_and_resize->y_bottoms_);
38 crop_and_resize->y_tops_ = (int *)env->Alloc(env->allocator_, height_size * sizeof(int));
39 NNACL_MALLOC_CHECK_NULL_RETURN_ERR(crop_and_resize->y_tops_);
40 crop_and_resize->y_bottom_weights_ = (float *)env->Alloc(env->allocator_, height_size * sizeof(float));
41 NNACL_MALLOC_CHECK_NULL_RETURN_ERR(crop_and_resize->y_bottom_weights_);
42
43 NNACL_CHECK_INT_MUL_NOT_OVERFLOW(crop_and_resize->new_width_, crop_and_resize->batch_, NNACL_ERR);
44 int width_size = crop_and_resize->new_width_ * crop_and_resize->batch_;
45 NNACL_CHECK_MALLOC_SIZE(width_size);
46 crop_and_resize->x_lefts_ = (int *)env->Alloc(env->allocator_, width_size * sizeof(int));
47 NNACL_MALLOC_CHECK_NULL_RETURN_ERR(crop_and_resize->x_lefts_);
48 crop_and_resize->x_rights_ = (int *)env->Alloc(env->allocator_, width_size * sizeof(int));
49 NNACL_MALLOC_CHECK_NULL_RETURN_ERR(crop_and_resize->x_rights_);
50 crop_and_resize->x_left_weights_ = (float *)env->Alloc(env->allocator_, width_size * sizeof(float));
51 NNACL_MALLOC_CHECK_NULL_RETURN_ERR(crop_and_resize->x_left_weights_);
52
53 int c = GetChannel(input_tensor);
54 NNACL_CHECK_INT_MUL_NOT_OVERFLOW(crop_and_resize->new_width_, c, NNACL_ERR);
55 int new_wc = crop_and_resize->new_width_ * c;
56 NNACL_CHECK_INT_MUL_NOT_OVERFLOW(new_wc, crop_and_resize->mapped_point_num_, NNACL_ERR);
57 int total_point_num = new_wc * crop_and_resize->mapped_point_num_;
58 NNACL_CHECK_INT_MUL_NOT_OVERFLOW(total_point_num, crop_and_resize->base_.thread_nr_, NNACL_ERR);
59 int line_buffer_size = total_point_num * crop_and_resize->base_.thread_nr_ * sizeof(float);
60 crop_and_resize->line_buffer_ = (float *)env->Alloc(env->allocator_, line_buffer_size);
61 NNACL_MALLOC_CHECK_NULL_RETURN_ERR(crop_and_resize->line_buffer_);
62 return NNACL_OK;
63 }
64
CropAndResizeFreeTmpBuffer(CropAndResizeStruct * crop_and_resize)65 void CropAndResizeFreeTmpBuffer(CropAndResizeStruct *crop_and_resize) {
66 ExecEnv *env = crop_and_resize->base_.env_;
67 NNACL_CHECK_NULL_RETURN_VOID(env);
68 env->Free(env->allocator_, crop_and_resize->y_bottoms_);
69 env->Free(env->allocator_, crop_and_resize->y_tops_);
70 env->Free(env->allocator_, crop_and_resize->y_bottom_weights_);
71 env->Free(env->allocator_, crop_and_resize->x_lefts_);
72 env->Free(env->allocator_, crop_and_resize->x_rights_);
73 env->Free(env->allocator_, crop_and_resize->x_left_weights_);
74 env->Free(env->allocator_, crop_and_resize->line_buffer_);
75 crop_and_resize->y_bottoms_ = NULL;
76 crop_and_resize->y_tops_ = NULL;
77 crop_and_resize->y_bottom_weights_ = NULL;
78 crop_and_resize->x_lefts_ = NULL;
79 crop_and_resize->x_rights_ = NULL;
80 crop_and_resize->x_left_weights_ = NULL;
81 crop_and_resize->line_buffer_ = NULL;
82 }
83
CropAndResizeImpl(void * cdata,int task_id,float l,float r)84 int CropAndResizeImpl(void *cdata, int task_id, float l, float r) {
85 CropAndResizeStruct *crop_and_resize = (CropAndResizeStruct *)cdata;
86 NNACL_CHECK_NULL_RETURN_ERR(crop_and_resize);
87
88 TensorC *input = crop_and_resize->base_.in_[FIRST_INPUT];
89 NNACL_CHECK_NULL_RETURN_ERR(input);
90 TensorC *boxes = crop_and_resize->base_.in_[SECOND_INPUT];
91 NNACL_CHECK_NULL_RETURN_ERR(boxes);
92 TensorC *box_idx = crop_and_resize->base_.in_[THIRD_INPUT];
93 NNACL_CHECK_NULL_RETURN_ERR(box_idx);
94 TensorC *output = crop_and_resize->base_.out_[OUTPUT_INDEX];
95 NNACL_CHECK_NULL_RETURN_ERR(output);
96
97 int unit = UP_DIV(crop_and_resize->new_height_, crop_and_resize->base_.thread_nr_);
98 NNACL_CHECK_INT_MUL_NOT_OVERFLOW(unit, task_id, NNACL_ERR);
99 int h_begin = unit * task_id;
100 int h_end = MSMIN(h_begin + unit, crop_and_resize->new_height_);
101 if (h_end <= h_begin) {
102 return NNACL_OK;
103 }
104
105 float extrapolation_value = ((CropAndResizeParameter *)crop_and_resize->base_.param_)->extrapolation_value_;
106 int c = input->shape_[kNHWC_C];
107 float *line0 = crop_and_resize->line_buffer_ + crop_and_resize->new_width_ * c * 2 * task_id;
108 float *line1 = line0 + crop_and_resize->new_width_ * c;
109
110 return CropAndResizeBilinear((float *)input->data_, (float *)output->data_, (int32_t *)box_idx->data_,
111 (float *)boxes->data_, extrapolation_value, input->shape_, output->shape_,
112 crop_and_resize->y_bottoms_, crop_and_resize->y_tops_, crop_and_resize->x_lefts_,
113 crop_and_resize->x_rights_, crop_and_resize->y_bottom_weights_,
114 crop_and_resize->x_left_weights_, line0, line1, h_begin, h_end);
115 }
116
CropAndResizeCompute(struct KernelBase * self)117 int CropAndResizeCompute(struct KernelBase *self) {
118 CropAndResizeStruct *crop_and_resize = (CropAndResizeStruct *)self;
119 NNACL_CHECK_NULL_RETURN_ERR(crop_and_resize);
120
121 // In Prepare() stage, in_tensor[0] may be of fp16 data type in fp16 mode, so move type checks here.
122 TensorC *input_tensor = self->in_[FIRST_INPUT];
123 NNACL_CHECK_NULL_RETURN_ERR(input_tensor);
124 TensorC *boxes_tensor = self->in_[SECOND_INPUT];
125 NNACL_CHECK_NULL_RETURN_ERR(boxes_tensor);
126 TensorC *boxidx_tensor = self->in_[THIRD_INPUT];
127 NNACL_CHECK_NULL_RETURN_ERR(boxidx_tensor);
128 TensorC *output_tensor = self->out_[OUTPUT_INDEX];
129 NNACL_CHECK_NULL_RETURN_ERR(output_tensor);
130
131 int ret = CropAndResizeMallocTmpBuffer(crop_and_resize);
132 if (ret != NNACL_OK) {
133 CropAndResizeFreeTmpBuffer(crop_and_resize);
134 return ret;
135 }
136
137 float *boxes = (float *)boxes_tensor->data_;
138 NNACL_CHECK_NULL_RETURN_ERR(boxes);
139 int32_t *box_idx = (int32_t *)boxidx_tensor->data_;
140 NNACL_CHECK_NULL_RETURN_ERR(box_idx);
141
142 if (CheckCropAndResizeBoxIdx(box_idx, boxes_tensor->shape_[Index0], GetBatch(input_tensor)) != NNACL_OK) {
143 return NNACL_CROP_AND_RESIZE_BOX_IDX_INVALID;
144 }
145
146 ret = PrepareCropAndResizeBilinear(input_tensor->shape_, boxes, box_idx, output_tensor->shape_,
147 crop_and_resize->y_bottoms_, crop_and_resize->y_tops_, crop_and_resize->x_lefts_,
148 crop_and_resize->x_rights_, crop_and_resize->y_bottom_weights_,
149 crop_and_resize->x_left_weights_);
150 if (ret != NNACL_OK) {
151 CropAndResizeFreeTmpBuffer(crop_and_resize);
152 return ret;
153 }
154
155 int error_code = self->env_->ParallelLaunch(self->env_->thread_pool_, CropAndResizeImpl, self, self->thread_nr_);
156 CropAndResizeFreeTmpBuffer(crop_and_resize);
157 return error_code;
158 }
159
CropAndResizeResize(KernelBase * self)160 int CropAndResizeResize(KernelBase *self) {
161 CropAndResizeStruct *crop_and_resize = (CropAndResizeStruct *)self;
162 NNACL_CHECK_NULL_RETURN_ERR(crop_and_resize);
163 TensorC *output = self->out_[FIRST_INPUT];
164 NNACL_CHECK_NULL_RETURN_ERR(output);
165 crop_and_resize->new_height_ = output->shape_[Index1];
166 crop_and_resize->new_width_ = output->shape_[Index2];
167 return NNACL_OK;
168 }
169
CropAndResizePrepare(KernelBase * self)170 int CropAndResizePrepare(KernelBase *self) {
171 NNACL_CHECK_FALSE(self->in_size_ < THREE_TENSOR, NNACL_ERR);
172 NNACL_CHECK_FALSE(self->out_size_ < ONE_TENSOR, NNACL_ERR);
173 NNACL_CHECK_NULL_RETURN_ERR(self->in_[FIRST_INPUT]);
174 NNACL_CHECK_NULL_RETURN_ERR(self->out_[OUTPUT_INDEX]);
175 return NNACL_OK;
176 }
177
CreateCropAndResize(OpParameter * param,int data_type)178 KernelBase *CreateCropAndResize(OpParameter *param, int data_type) {
179 CropAndResizeStruct *crop_and_resize = (CropAndResizeStruct *)malloc(sizeof(CropAndResizeStruct));
180 NNACL_MALLOC_CHECK_NULL_RETURN_NULL(crop_and_resize);
181 memset(crop_and_resize, 0, sizeof(CropAndResizeStruct));
182 crop_and_resize->mapped_point_num_ = Num2;
183 crop_and_resize->base_.Prepare = CropAndResizePrepare;
184 crop_and_resize->base_.Resize = CropAndResizeResize;
185 crop_and_resize->base_.Compute = CropAndResizeCompute;
186 crop_and_resize->base_.Release = DefaultRelease;
187 return (KernelBase *)crop_and_resize;
188 }
189
190 REG_KERNEL_CREATOR(PrimType_CropAndResize, kNumberTypeFloat32, CreateCropAndResize)
191