1 /*
2 * Copyright (c) 2019-2020 Arm Limited.
3 *
4 * SPDX-License-Identifier: MIT
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to
8 * deal in the Software without restriction, including without limitation the
9 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
10 * sell copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in all
14 * copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 * SOFTWARE.
23 */
24 #include "src/core/CL/kernels/CLGEMMMatrixMultiplyReshapedOnlyRHSKernel.h"
25
26 #include "arm_compute/core/CL/ICLTensor.h"
27 #include "arm_compute/core/Helpers.h"
28 #include "arm_compute/core/TensorInfo.h"
29 #include "arm_compute/core/Utils.h"
30 #include "arm_compute/core/utils/misc/ShapeCalculator.h"
31 #include "src/core/AccessWindowStatic.h"
32 #include "src/core/CL/CLUtils.h"
33 #include "src/core/CL/CLValidate.h"
34 #include "src/core/CL/gemm/CLGEMMHelpers.h"
35 #include "src/core/helpers/AutoConfiguration.h"
36 #include "src/core/helpers/WindowHelpers.h"
37 #include "src/core/utils/helpers/float_ops.h"
38 #include "support/StringSupport.h"
39
40 #include <tuple>
41
42 using namespace arm_compute::misc::shape_calculator;
43
44 namespace arm_compute
45 {
46 namespace
47 {
48 using ElementsProcessed = Steps;
49
validate_arguments(const ITensorInfo * input0,const ITensorInfo * input1,const ITensorInfo * input2,const ITensorInfo * output,float alpha,float beta,const GEMMLHSMatrixInfo & lhs_info,const GEMMRHSMatrixInfo & rhs_info,const GEMMKernelInfo & gemm_info)50 Status validate_arguments(const ITensorInfo *input0, const ITensorInfo *input1, const ITensorInfo *input2, const ITensorInfo *output, float alpha, float beta, const GEMMLHSMatrixInfo &lhs_info,
51 const GEMMRHSMatrixInfo &rhs_info,
52 const GEMMKernelInfo &gemm_info)
53 {
54 ARM_COMPUTE_UNUSED(alpha);
55 ARM_COMPUTE_RETURN_ERROR_ON_NULLPTR(input0, input1, output);
56 ARM_COMPUTE_RETURN_ERROR_ON_F16_UNSUPPORTED(input0);
57 ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(input0, 1, DataType::F16, DataType::F32);
58 ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(input0, input1);
59 ARM_COMPUTE_RETURN_ERROR_ON_MSG(input0->num_dimensions() > 4, "The number of dimensions for the LHS matrix must be <= 4");
60 ARM_COMPUTE_RETURN_ERROR_ON_MSG(input1->num_dimensions() > 3, "The number of dimensions for the RHS matrix must be <= 3");
61 ARM_COMPUTE_RETURN_ERROR_ON_MSG(lhs_info.m0 < 1 || lhs_info.m0 > 8, "Only 1,2,3,4,5,6,7,8 are supported for m0");
62 ARM_COMPUTE_RETURN_ERROR_ON(rhs_info.k0 > 16 || rhs_info.k0 < 2);
63 ARM_COMPUTE_RETURN_ERROR_ON_MSG(((rhs_info.k0 & (rhs_info.k0 - 1)) && rhs_info.k0 != 3), "Only 2,3,4,8,16 are supported for k0");
64 ARM_COMPUTE_RETURN_ERROR_ON(rhs_info.n0 > 16 || rhs_info.n0 < 2);
65 ARM_COMPUTE_RETURN_ERROR_ON_MSG(((rhs_info.n0 & (rhs_info.n0 - 1)) && rhs_info.n0 != 3), "Only 2,3,4,8,16 are supported for n0");
66 ARM_COMPUTE_RETURN_ERROR_ON_MSG((gemm_info.reinterpret_input_as_3d || gemm_info.depth_output_gemm3d != 0) && (input2 != nullptr)
67 && (!gemm_info.broadcast_bias),
68 "Bias addition only supported with broadcast mode in case the input or output has to be reinterpreted as 3D");
69 ARM_COMPUTE_RETURN_ERROR_ON_MSG(gemm_info.fp_mixed_precision, "Mixed precision not supported");
70 ARM_COMPUTE_RETURN_ON_ERROR(cl_gemm::validate_image2d_support_on_rhs(*input1, rhs_info));
71
72 const unsigned int m = gemm_info.m;
73 const unsigned int n = gemm_info.n;
74 const unsigned int k = gemm_info.k;
75
76 TensorShape tensor_shape1{ input1->tensor_shape() };
77 tensor_shape1.set(0, n);
78 tensor_shape1.set(1, k);
79
80 if(input2 != nullptr && !(helpers::float_ops::is_zero(beta)))
81 {
82 const unsigned int input2_dim0 = input2->dimension(0);
83 const unsigned int input2_dim1 = input2->dimension(1);
84
85 ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(input2, input0);
86 if(gemm_info.broadcast_bias)
87 {
88 ARM_COMPUTE_RETURN_ERROR_ON_MSG((input2_dim1 != 1 || input2_dim0 != n), "Incorrect dimension of bias matrix which is to be broadcasted");
89 }
90 else
91 {
92 ARM_COMPUTE_RETURN_ERROR_ON_MSG((input2_dim0 != n || input2_dim1 != m), "Incorrect dimension of bias matrix");
93 }
94 }
95
96 const TensorInfo tensor_info1 = input1->clone()->set_tensor_shape(tensor_shape1);
97
98 const TensorInfo tensor_info_reshaped1 = input1->clone()->set_tensor_shape(compute_rhs_reshaped_shape(tensor_info1, rhs_info));
99
100 ARM_COMPUTE_RETURN_ERROR_ON(input0->dimension(0) != k);
101 if(gemm_info.reinterpret_input_as_3d)
102 {
103 ARM_COMPUTE_RETURN_ERROR_ON(input0->dimension(1) * input0->dimension(2) != m);
104 }
105 else
106 {
107 ARM_COMPUTE_RETURN_ERROR_ON(input0->dimension(1) != m);
108 }
109 ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_SHAPES(input1, &tensor_info_reshaped1);
110
111 if(output->total_size() != 0)
112 {
113 const TensorInfo tensor_info_output = output->clone()->set_tensor_shape(compute_mm_shape(*input0, *input1, gemm_info));
114 ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_SHAPES(output, &tensor_info_output);
115 ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(input0, output);
116 }
117
118 return Status{};
119 }
120
validate_and_configure_window(ITensorInfo * input0,ITensorInfo * input1,ITensorInfo * input2,ITensorInfo * output,const GEMMLHSMatrixInfo & lhs_info,const GEMMRHSMatrixInfo & rhs_info,const GEMMKernelInfo & gemm_info,ElementsProcessed & num_elements_processed)121 std::pair<Status, Window> validate_and_configure_window(ITensorInfo *input0, ITensorInfo *input1, ITensorInfo *input2, ITensorInfo *output, const GEMMLHSMatrixInfo &lhs_info,
122 const GEMMRHSMatrixInfo &rhs_info,
123 const GEMMKernelInfo &gemm_info, ElementsProcessed &num_elements_processed)
124 {
125 unsigned int &num_elems_processed_per_iteration_x = num_elements_processed[0];
126 unsigned int &num_elems_processed_per_iteration_y = num_elements_processed[1];
127 bool reinterpret_input_as_3d = gemm_info.reinterpret_input_as_3d;
128 bool reinterpret_output_as_3d = gemm_info.depth_output_gemm3d != 0;
129
130 Window win{};
131 Window win_out{};
132 bool window_changed = false;
133
134 // In case both input and output have to be reinterpreted as 3D tensors,
135 // force reinterpret_input_as_3d and reinterpret_output_as_3d to be false.
136 // This approach should only be used when the input/output tensors have pad on the y direction
137 if((reinterpret_input_as_3d == reinterpret_output_as_3d) && gemm_info.has_pad_y)
138 {
139 reinterpret_output_as_3d = false;
140 }
141
142 // Output tensor auto initialization if not yet initialized
143 auto_init_if_empty(*output, input0->clone()->set_tensor_shape(compute_mm_shape(*input0, *input1, gemm_info)));
144
145 TensorInfo tmp_info(*output);
146
147 if(reinterpret_output_as_3d)
148 {
149 // Since the output tensor has to be reinterpreted as 3D and the execute window is based on a 2D GEMM,
150 // the window needs to be constructed on the 2D collapsed version of the tensor
151 TensorShape tmp_shape(output->tensor_shape());
152 tmp_shape.collapse(2U, 1U);
153 tmp_info.set_tensor_shape(tmp_shape);
154 }
155
156 // Configure kernel window
157 num_elems_processed_per_iteration_x = rhs_info.n0;
158 num_elems_processed_per_iteration_y = lhs_info.m0;
159
160 win = calculate_max_window(tmp_info, Steps(num_elems_processed_per_iteration_x, num_elems_processed_per_iteration_y));
161 win_out = calculate_max_window(*output, Steps(num_elems_processed_per_iteration_x, num_elems_processed_per_iteration_y));
162
163 if(input2 != nullptr)
164 {
165 const int bias_processed_per_iteration_x = num_elems_processed_per_iteration_x;
166
167 AccessWindowStatic input2_access(input2, 0, 0,
168 ceil_to_multiple(input2->dimension(0), bias_processed_per_iteration_x),
169 input2->dimension(1));
170
171 window_changed = update_window_and_padding(win, input2_access);
172 }
173
174 // Collapse along the Z direction
175 // This collapse needs to be here in order to tune the Z dimension of LWS
176 Window collapsed = win;
177 const unsigned int dimension_to_collapse = std::min(static_cast<unsigned int>(output->num_dimensions()), 2u);
178 collapsed = win.collapse(win, dimension_to_collapse);
179
180 Status err = (window_changed) ? ARM_COMPUTE_CREATE_ERROR(ErrorCode::RUNTIME_ERROR, "Insufficient Padding!") : Status{};
181 return std::make_pair(err, collapsed);
182 }
183 } // namespace
184
CLGEMMMatrixMultiplyReshapedOnlyRHSKernel()185 CLGEMMMatrixMultiplyReshapedOnlyRHSKernel::CLGEMMMatrixMultiplyReshapedOnlyRHSKernel()
186 : _input0(nullptr), _input1(nullptr), _input2(nullptr), _output(nullptr), _slide_matrix_b(true), _reinterpret_input_as_3d(false), _reinterpret_output_as_3d(false), _use_dummy_work_items(false),
187 _add_bias(false), _broadcast_bias(false), _export_to_cl_image(false), _has_pad_y(false)
188 {
189 }
190
configure(const ICLTensor * input0,const ICLTensor * input1,const ICLTensor * input2,ICLTensor * output,float alpha,float beta,const GEMMLHSMatrixInfo & lhs_info,const GEMMRHSMatrixInfo & rhs_info,const GEMMKernelInfo & gemm_info)191 void CLGEMMMatrixMultiplyReshapedOnlyRHSKernel::configure(const ICLTensor *input0, const ICLTensor *input1, const ICLTensor *input2, ICLTensor *output, float alpha, float beta,
192 const GEMMLHSMatrixInfo &lhs_info,
193 const GEMMRHSMatrixInfo &rhs_info, const GEMMKernelInfo &gemm_info)
194 {
195 configure(CLKernelLibrary::get().get_compile_context(), input0, input1, input2, output, alpha, beta, lhs_info, rhs_info, gemm_info);
196 }
197
configure(const CLCompileContext & compile_context,const ICLTensor * input0,const ICLTensor * input1,const ICLTensor * input2,ICLTensor * output,float alpha,float beta,const GEMMLHSMatrixInfo & lhs_info,const GEMMRHSMatrixInfo & rhs_info,const GEMMKernelInfo & gemm_info)198 void CLGEMMMatrixMultiplyReshapedOnlyRHSKernel::configure(const CLCompileContext &compile_context, const ICLTensor *input0, const ICLTensor *input1, const ICLTensor *input2, ICLTensor *output,
199 float alpha,
200 float beta,
201 const GEMMLHSMatrixInfo &lhs_info,
202 const GEMMRHSMatrixInfo &rhs_info, const GEMMKernelInfo &gemm_info)
203 {
204 ARM_COMPUTE_ERROR_ON_NULLPTR(input0, input1, output);
205
206 ARM_COMPUTE_ERROR_THROW_ON(validate_arguments(input0->info(), input1->info(), (input2 != nullptr ? input2->info() : nullptr), output->info(), alpha, beta, lhs_info, rhs_info, gemm_info));
207
208 _input0 = input0;
209 _input1 = input1;
210 _input2 = helpers::float_ops::is_zero(beta) ? nullptr : input2;
211 _output = output;
212 _reinterpret_input_as_3d = gemm_info.reinterpret_input_as_3d;
213 _reinterpret_output_as_3d = gemm_info.depth_output_gemm3d != 0;
214 _use_dummy_work_items = preferred_dummy_work_items_support(CLKernelLibrary::get().get_device());
215 _add_bias = _input2 != nullptr;
216 _broadcast_bias = gemm_info.broadcast_bias;
217 _export_to_cl_image = rhs_info.export_to_cl_image;
218 _has_pad_y = gemm_info.has_pad_y;
219
220 auto padding_info = get_padding_info({ input0, input1, output });
221
222 // In case both input and output have to be reinterpreted as 3D tensors,
223 // force reinterpret_input_as_3d and reinterpret_output_as_3d to be false.
224 if((_reinterpret_input_as_3d == _reinterpret_output_as_3d) && _has_pad_y)
225 {
226 _reinterpret_input_as_3d = false;
227 _reinterpret_output_as_3d = false;
228 }
229
230 // Check if we need to slide the matrix B
231 const unsigned int num_dimensions_input0 = _input0->info()->num_dimensions();
232 _slide_matrix_b = (_input1->info()->num_dimensions() >= num_dimensions_input0);
233
234 ElementsProcessed num_elements_processed{};
235
236 // Configure kernel window
237 auto win_config = validate_and_configure_window(input0->info(), input1->info(), input2 != nullptr ? input2->info() : nullptr, output->info(), lhs_info, rhs_info, gemm_info, num_elements_processed);
238 ARM_COMPUTE_ERROR_THROW_ON(win_config.first);
239 ICLKernel::configure_internal(win_config.second);
240
241 // If _reinterpret_input_as_3d = _reinterpret_output_as_3d = true,
242 // we will dispatch a batched-GEMM to reduce the complexity of the address calculation within the OpenCL kernel.
243 // This means that the actual m used by the kernel is given by output->info()->dimension(1) and not by gemm_info.m
244 const unsigned int internal_m = _reinterpret_output_as_3d ? gemm_info.m : output->info()->dimension(1);
245
246 // These variables are used only if gemm_info.has_pad_y == true
247 const unsigned int h_gemm_3d = _reinterpret_output_as_3d ? output->info()->dimension(1) : input0->info()->dimension(1);
248 const unsigned int d_gemm_3d = _reinterpret_output_as_3d ? output->info()->dimension(2) : input0->info()->dimension(2);
249
250 // Shrink M0 to be always <= M (internal_m) to prevent out-of-bounds reads.
251 // NOTE: This might have implications on heuristics and performance
252 const unsigned int internal_m0 = std::min(internal_m, lhs_info.m0);
253
254 // Calculate partial (store instead of load) M0 and partial N0 for the partial blocks at the end of a row/column if any. This is to avoid padding.
255 const unsigned int partial_store_m0 = internal_m % internal_m0;
256 const unsigned int partial_store_n0 = gemm_info.n % rhs_info.n0;
257
258 // Create build options
259 CLBuildOptions build_opts;
260 build_opts.add_option("-DDATA_TYPE=" + get_cl_type_from_data_type(input0->info()->data_type()));
261 build_opts.add_option_if(!(helpers::float_ops::is_one(alpha)), "-DALPHA=" + float_to_string_with_full_precision(alpha));
262 build_opts.add_option_if(_input2 != nullptr, "-DBETA=" + float_to_string_with_full_precision(beta));
263 build_opts.add_option_if(helpers::float_ops::is_one(beta), "-DUNIT_BETA");
264 build_opts.add_option_if(gemm_info.broadcast_bias, "-DBROADCAST_BIAS");
265 build_opts.add_option_if(!_slide_matrix_b, "-DMATRIX_B_DEPTH=" + support::cpp11::to_string(input1->info()->dimension(2)));
266 build_opts.add_option_if(rhs_info.interleave, "-DRHS_INTERLEAVE");
267 build_opts.add_option_if(_use_dummy_work_items, "-DDUMMY_WORK_ITEMS");
268 build_opts.add_option_if(rhs_info.export_to_cl_image, "-DOPENCL_IMAGE_SUPPORT");
269 build_opts.add_option("-DRHS_HEIGHT=" + support::cpp11::to_string(input1->info()->dimension(1)));
270 build_opts.add_option("-DM=" + support::cpp11::to_string(internal_m));
271 build_opts.add_option("-DN=" + support::cpp11::to_string(gemm_info.n));
272 build_opts.add_option("-DK=" + support::cpp11::to_string(gemm_info.k));
273 build_opts.add_option("-DM0=" + support::cpp11::to_string(internal_m0));
274 build_opts.add_option("-DN0=" + support::cpp11::to_string(rhs_info.n0));
275 build_opts.add_option("-DK0=" + support::cpp11::to_string(rhs_info.k0));
276 build_opts.add_option("-DH0=" + support::cpp11::to_string(rhs_info.h0));
277 build_opts.add_option("-DPARTIAL_STORE_M0=" + support::cpp11::to_string(partial_store_m0));
278 build_opts.add_option("-DPARTIAL_STORE_N0=" + support::cpp11::to_string(partial_store_n0));
279 build_opts.add_option_if(gemm_info.activation_info.enabled(), "-DACTIVATION_TYPE=" + lower_string(string_from_activation_func(gemm_info.activation_info.activation())));
280 build_opts.add_option_if(gemm_info.activation_info.enabled(), "-DA_VAL=" + float_to_string_with_full_precision(gemm_info.activation_info.a()));
281 build_opts.add_option_if(gemm_info.activation_info.enabled(), "-DB_VAL=" + float_to_string_with_full_precision(gemm_info.activation_info.b()));
282 if(_has_pad_y)
283 {
284 build_opts.add_option_if(_reinterpret_input_as_3d, "-DREINTERPRET_INPUT_AS_3D");
285 build_opts.add_option_if(_reinterpret_output_as_3d, "-DREINTERPRET_OUTPUT_AS_3D");
286 build_opts.add_option_if(_reinterpret_input_as_3d || _reinterpret_output_as_3d, "-DHEIGHT_GEMM3D=" + support::cpp11::to_string(h_gemm_3d));
287 build_opts.add_option_if(_reinterpret_input_as_3d || _reinterpret_output_as_3d, "-DDEPTH_GEMM3D=" + support::cpp11::to_string(d_gemm_3d));
288 }
289
290 std::string kernel_name("gemm_mm_reshaped_only_rhs_");
291 kernel_name += rhs_info.transpose ? "t" : "nt";
292 kernel_name += rhs_info.export_to_cl_image ? "_texture" : "";
293
294 // Create kernel
295 _kernel = create_kernel(compile_context, kernel_name, build_opts.options());
296
297 // Set config_id for enabling LWS tuning
298 _config_id = kernel_name;
299 _config_id += "_";
300 _config_id += (_has_pad_y ? "" : "no_pad_y_");
301 _config_id += (_add_bias ? "add_bias_" : "");
302 _config_id += (_broadcast_bias ? "broadcast_bias_" : "");
303 _config_id += (_reinterpret_input_as_3d ? "3di_" : "");
304 _config_id += (_reinterpret_output_as_3d ? "3do_" : "");
305 _config_id += (gemm_info.activation_info.enabled() ? "fused_activation_" : "");
306 _config_id += lower_string(string_from_data_type(input0->info()->data_type()));
307 _config_id += "_";
308 _config_id += support::cpp11::to_string(output->info()->dimension(1));
309 _config_id += "_";
310 _config_id += support::cpp11::to_string(output->info()->dimension(0));
311 _config_id += "_";
312 _config_id += support::cpp11::to_string(gemm_info.k);
313 _config_id += "_";
314 _config_id += support::cpp11::to_string(output->info()->dimension(2));
315 _config_id += "_";
316 _config_id += support::cpp11::to_string(lhs_info.m0);
317 _config_id += "_";
318 _config_id += support::cpp11::to_string(rhs_info.n0);
319 _config_id += "_";
320 _config_id += support::cpp11::to_string(rhs_info.k0);
321 _config_id += "_";
322 _config_id += support::cpp11::to_string(rhs_info.h0);
323 _config_id += "_";
324 _config_id += support::cpp11::to_string(rhs_info.interleave);
325
326 ARM_COMPUTE_ERROR_ON(has_padding_changed(padding_info));
327 }
328
validate(const ITensorInfo * input0,const ITensorInfo * input1,const ITensorInfo * input2,const ITensorInfo * output,float alpha,float beta,const GEMMLHSMatrixInfo & lhs_info,const GEMMRHSMatrixInfo & rhs_info,const GEMMKernelInfo & gemm_info)329 Status CLGEMMMatrixMultiplyReshapedOnlyRHSKernel::validate(const ITensorInfo *input0, const ITensorInfo *input1, const ITensorInfo *input2, const ITensorInfo *output, float alpha, float beta,
330 const GEMMLHSMatrixInfo &lhs_info,
331 const GEMMRHSMatrixInfo &rhs_info, const GEMMKernelInfo &gemm_info)
332 {
333 ElementsProcessed num_elements_processed{};
334 ARM_COMPUTE_RETURN_ON_ERROR(validate_arguments(input0, input1, input2, output, alpha, beta, lhs_info, rhs_info, gemm_info));
335 ARM_COMPUTE_RETURN_ON_ERROR(validate_and_configure_window(input0->clone().get(),
336 input1->clone().get(),
337 input2 != nullptr ? input2->clone().get() : nullptr,
338 output->clone().get(),
339 lhs_info,
340 rhs_info,
341 gemm_info,
342 num_elements_processed)
343 .first);
344
345 return Status{};
346 }
347
run(const Window & window,cl::CommandQueue & queue)348 void CLGEMMMatrixMultiplyReshapedOnlyRHSKernel::run(const Window &window, cl::CommandQueue &queue)
349 {
350 ARM_COMPUTE_ERROR_ON_UNCONFIGURED_KERNEL(this);
351 ARM_COMPUTE_ERROR_ON_INVALID_SUBWINDOW(ICLKernel::window(), window);
352
353 if(_input1->info()->num_dimensions() < 3)
354 {
355 // The stride_z for matrix B must be zero if we do not slice
356 ARM_COMPUTE_ERROR_ON(_input1->info()->strides_in_bytes()[3] != 0);
357 }
358
359 const size_t lhs_idx_batch_size = _reinterpret_input_as_3d && !_has_pad_y ? 3u : 2u;
360 const size_t rhs_idx_batch_size = 2u;
361 const size_t bia_idx_batch_size = 2u;
362 const size_t out_idx_batch_size = _reinterpret_output_as_3d && !_has_pad_y ? 3u : 2u;
363
364 Window slice = window.first_slice_window_3D();
365 Window slice_matrix_b = slice;
366
367 slice_matrix_b.set(Window::DimX, Window::Dimension(0, 1, 1));
368 slice_matrix_b.set(Window::DimY, Window::Dimension(0, 1, 1));
369
370 // Get cross plane pads
371 const unsigned int total_cross_plane_pad_lhs = _input0->info()->padding().top + _input0->info()->padding().bottom;
372 const unsigned int total_cross_plane_pad_out = _output->info()->padding().top + _output->info()->padding().bottom;
373
374 // The execution should fail if we try to run with has_pad_y = false but we have padding in either the LHS or DST tensor
375 ARM_COMPUTE_ERROR_ON(!_has_pad_y && ((total_cross_plane_pad_lhs != 0) || (total_cross_plane_pad_out != 0)));
376
377 cl::Image2D input1_image2d;
378
379 if(_export_to_cl_image)
380 {
381 const TensorShape shape2d(_input1->info()->dimension(0) / 4, _input1->info()->dimension(1) * _input1->info()->dimension(2));
382 const size_t image_row_pitch = _input1->info()->strides_in_bytes()[1];
383
384 input1_image2d = create_image2d_from_buffer(CLKernelLibrary::get().context(), _input1->cl_buffer(), shape2d, _input1->info()->data_type(), image_row_pitch);
385 }
386
387 do
388 {
389 Window slice_b = slice;
390 // Don't slice matrix B along the z dimension if matrix B has just 2 dimensions and matrix A more than 2
391 // This scenario can happen when the matrix multiplication is used to perform a convolution operation
392 if(!_slide_matrix_b)
393 {
394 slice_b = slice_matrix_b;
395 }
396
397 unsigned int idx = 0;
398
399 // LHS buffer
400 add_2D_tensor_argument(idx, _input0, slice);
401
402 // RHS buffer or RHS OpenCL image (_export_to_cl_image == true)
403 if(_export_to_cl_image)
404 {
405 _kernel.setArg(idx++, input1_image2d);
406 }
407 else
408 {
409 add_2D_tensor_argument(idx, _input1, slice_b);
410 }
411
412 // Bias buffer (_add_bias == true)
413 add_2D_tensor_argument_if(_add_bias, idx, _input2, slice);
414
415 // Output buffer
416 add_2D_tensor_argument(idx, _output, slice);
417
418 // LHS stride_z
419 _kernel.setArg<cl_uint>(idx++, static_cast<unsigned int>(_input0->info()->strides_in_bytes()[lhs_idx_batch_size]));
420
421 // RHS stride_z (not used if _export_to_cl_image == true)
422 _kernel.setArg<cl_uint>(idx++, static_cast<unsigned int>(_input1->info()->strides_in_bytes()[rhs_idx_batch_size]));
423
424 // Bias stride_z (if _add_bias == true)
425 if(_add_bias)
426 {
427 _kernel.setArg<cl_uint>(idx++, static_cast<unsigned int>(_input2->info()->strides_in_bytes()[bia_idx_batch_size]));
428 }
429
430 // Output stride_z
431 _kernel.setArg<cl_uint>(idx++, static_cast<unsigned int>(_output->info()->strides_in_bytes()[out_idx_batch_size]));
432
433 // Cross-plan padding (if _reinterpret_input_as_3d = true)
434 if(_reinterpret_input_as_3d && _has_pad_y)
435 {
436 _kernel.setArg<cl_uint>(idx++, static_cast<unsigned int>(total_cross_plane_pad_lhs));
437 }
438
439 // Cross-plan padding (if _reinterpret_output_as_3d = true)
440 if(_reinterpret_output_as_3d && _has_pad_y)
441 {
442 _kernel.setArg<cl_uint>(idx++, static_cast<unsigned int>(total_cross_plane_pad_out));
443 }
444
445 enqueue(queue, *this, slice, lws_hint(), _use_dummy_work_items);
446 }
447 while(window.slide_window_slice_3D(slice));
448 }
449 } // namespace arm_compute
450