1 /*
2 * Copyright (c) 2018-2020 Arm Limited.
3 *
4 * SPDX-License-Identifier: MIT
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to
8 * deal in the Software without restriction, including without limitation the
9 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
10 * sell copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in all
14 * copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 * SOFTWARE.
23 */
24 #include "src/core/CL/kernels/CLGEMMMatrixMultiplyReshapedKernel.h"
25
26 #include "arm_compute/core/CL/CLHelpers.h"
27 #include "arm_compute/core/CL/CLKernelLibrary.h"
28 #include "arm_compute/core/CL/ICLTensor.h"
29 #include "arm_compute/core/CL/OpenCL.h"
30 #include "arm_compute/core/Helpers.h"
31 #include "arm_compute/core/TensorInfo.h"
32 #include "arm_compute/core/Utils.h"
33 #include "arm_compute/core/Validate.h"
34 #include "arm_compute/core/utils/misc/ShapeCalculator.h"
35 #include "src/core/AccessWindowStatic.h"
36 #include "src/core/CL/CLUtils.h"
37 #include "src/core/CL/CLValidate.h"
38 #include "src/core/CL/gemm/CLGEMMHelpers.h"
39 #include "src/core/helpers/AutoConfiguration.h"
40 #include "src/core/helpers/WindowHelpers.h"
41 #include "src/core/utils/helpers/float_ops.h"
42 #include "support/StringSupport.h"
43
44 #include <cstddef>
45 #include <cstdint>
46 #include <tuple>
47
48 using namespace arm_compute;
49 using namespace arm_compute::misc::shape_calculator;
50
51 namespace arm_compute
52 {
53 class Coordinates;
54 } // namespace arm_compute
55
56 namespace
57 {
58 using ElementsProcessed = Steps;
59
validate_arguments(const ITensorInfo * input0,const ITensorInfo * input1,const ITensorInfo * input2,const ITensorInfo * output,float alpha,float beta,const GEMMLHSMatrixInfo & lhs_info,const GEMMRHSMatrixInfo & rhs_info,const GEMMKernelInfo & gemm_info)60 Status validate_arguments(const ITensorInfo *input0, const ITensorInfo *input1, const ITensorInfo *input2, const ITensorInfo *output, float alpha, float beta, const GEMMLHSMatrixInfo &lhs_info,
61 const GEMMRHSMatrixInfo &rhs_info,
62 const GEMMKernelInfo &gemm_info)
63 {
64 ARM_COMPUTE_UNUSED(alpha);
65 ARM_COMPUTE_RETURN_ERROR_ON_NULLPTR(input0, input1, output);
66 ARM_COMPUTE_RETURN_ERROR_ON_F16_UNSUPPORTED(input0);
67 ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(input0, 1, DataType::F16, DataType::F32);
68 ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(input0, input1);
69 ARM_COMPUTE_RETURN_ERROR_ON_MSG(input0->num_dimensions() > 4, "The number of dimensions for the LHS matrix must be <= 4");
70 ARM_COMPUTE_RETURN_ERROR_ON_MSG(input1->num_dimensions() > 3, "The number of dimensions for the RHS matrix must be <= 3");
71 ARM_COMPUTE_RETURN_ERROR_ON(lhs_info.k0 != rhs_info.k0);
72 ARM_COMPUTE_RETURN_ERROR_ON(lhs_info.transpose == rhs_info.transpose);
73 ARM_COMPUTE_RETURN_ERROR_ON_MSG(((lhs_info.k0 & (lhs_info.k0 - 1)) && lhs_info.k0 != 3), "Only 2,3,4,8,16 are supported for k0");
74 ARM_COMPUTE_RETURN_ERROR_ON(lhs_info.k0 > 16);
75 ARM_COMPUTE_RETURN_ERROR_ON(lhs_info.m0 < 2 || lhs_info.m0 > 8);
76 ARM_COMPUTE_RETURN_ERROR_ON_MSG((lhs_info.transpose) && ((lhs_info.m0 & (lhs_info.m0 - 1)) && lhs_info.m0 != 3), "Only 2,3,4,8,16 are supported for m0");
77 ARM_COMPUTE_RETURN_ERROR_ON_MSG((rhs_info.transpose) && ((rhs_info.n0 & (rhs_info.n0 - 1)) && rhs_info.n0 != 3), "Only 2,3,4,8,16 are supported for n0");
78 ARM_COMPUTE_RETURN_ERROR_ON_MSG((gemm_info.reinterpret_input_as_3d || gemm_info.depth_output_gemm3d != 0) && (input2 != nullptr)
79 && (!gemm_info.broadcast_bias),
80 "Bias addition only supported with broadcast mode in case the input or output has to be reinterpreted as 3D");
81 ARM_COMPUTE_RETURN_ERROR_ON_MSG(gemm_info.fp_mixed_precision && (input0->data_type() == DataType::F32), "Mixed precision only supported for F16 data type");
82 ARM_COMPUTE_RETURN_ON_ERROR(cl_gemm::validate_image2d_support_on_rhs(*input1, rhs_info));
83
84 const unsigned int m = gemm_info.m;
85 const unsigned int n = gemm_info.n;
86 const unsigned int k = gemm_info.k;
87
88 TensorShape tensor_shape0{ input0->tensor_shape() };
89 tensor_shape0.set(0, k);
90 tensor_shape0.set(1, m);
91
92 TensorShape tensor_shape1{ input1->tensor_shape() };
93 tensor_shape1.set(0, n);
94 tensor_shape1.set(1, k);
95
96 if(input2 != nullptr && !(helpers::float_ops::is_zero(beta)))
97 {
98 const unsigned int input2_dim0 = input2->dimension(0);
99 const unsigned int input2_dim1 = input2->dimension(1);
100
101 ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(input2, input1);
102 if(gemm_info.broadcast_bias)
103 {
104 ARM_COMPUTE_RETURN_ERROR_ON_MSG((input2_dim1 != 1 || input2_dim0 != n), "Incorrect dimension of bias matrix which is to be broadcasted");
105 }
106 else
107 {
108 ARM_COMPUTE_RETURN_ERROR_ON_MSG((input2_dim0 != n || input2_dim1 != m), "Incorrect dimension of bias matrix");
109 }
110 }
111
112 const TensorInfo tensor_info0 = input0->clone()->set_tensor_shape(tensor_shape0);
113 const TensorInfo tensor_info1 = input1->clone()->set_tensor_shape(tensor_shape1);
114
115 const TensorInfo tensor_info_reshaped0 = input0->clone()->set_tensor_shape(compute_lhs_reshaped_shape(tensor_info0, lhs_info));
116 const TensorInfo tensor_info_reshaped1 = input1->clone()->set_tensor_shape(compute_rhs_reshaped_shape(tensor_info1, rhs_info));
117
118 ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_SHAPES(input0, &tensor_info_reshaped0);
119 ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_SHAPES(input1, &tensor_info_reshaped1);
120
121 if(output->total_size() != 0)
122 {
123 const TensorInfo tensor_info_output = output->clone()->set_tensor_shape(compute_mm_shape(*input0, *input1, gemm_info));
124 ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_SHAPES(output, &tensor_info_output);
125 ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(input0, output);
126 }
127
128 return Status{};
129 }
130
validate_and_configure_window(ITensorInfo * input0,ITensorInfo * input1,ITensorInfo * input2,ITensorInfo * output,const GEMMLHSMatrixInfo & lhs_info,const GEMMRHSMatrixInfo & rhs_info,const GEMMKernelInfo & gemm_info,ElementsProcessed & num_elements_processed)131 std::pair<Status, Window> validate_and_configure_window(ITensorInfo *input0, ITensorInfo *input1, ITensorInfo *input2, ITensorInfo *output, const GEMMLHSMatrixInfo &lhs_info,
132 const GEMMRHSMatrixInfo &rhs_info,
133 const GEMMKernelInfo &gemm_info, ElementsProcessed &num_elements_processed)
134 {
135 unsigned int &num_elems_processed_per_iteration_x = num_elements_processed[0];
136 unsigned int &num_elems_processed_per_iteration_y = num_elements_processed[1];
137 bool reinterpret_output_as_3d = gemm_info.depth_output_gemm3d != 0;
138
139 Window win{};
140 Window win_out{};
141 bool window_changed = false;
142
143 // Output tensor auto initialization if not yet initialized
144 auto_init_if_empty(*output, input0->clone()->set_tensor_shape(compute_mm_shape(*input0, *input1, gemm_info)));
145
146 TensorInfo tmp_info(*output);
147
148 if(reinterpret_output_as_3d)
149 {
150 // Since the output tensor has to be reinterpreted as 3D and the execute window is based on a 2D GEMM,
151 // the window needs to be constructed on the 2D collapsed version of the tensor
152 TensorShape tmp_shape(output->tensor_shape());
153 tmp_shape.collapse(2U, 1U);
154 tmp_info.set_tensor_shape(tmp_shape);
155 }
156
157 // Configure kernel window
158 num_elems_processed_per_iteration_x = rhs_info.n0;
159 num_elems_processed_per_iteration_y = lhs_info.m0;
160
161 win = calculate_max_window(tmp_info, Steps(num_elems_processed_per_iteration_x, num_elems_processed_per_iteration_y));
162 win_out = calculate_max_window(*output, Steps(num_elems_processed_per_iteration_x, num_elems_processed_per_iteration_y));
163
164 AccessWindowStatic input0_access(input0, 0, 0,
165 input0->dimension(0),
166 input0->dimension(1));
167 AccessWindowStatic input1_access(input1, 0, 0,
168 input1->dimension(0),
169 input1->dimension(1));
170 AccessWindowStatic output_access(output, 0, 0,
171 output->dimension(0),
172 output->dimension(1));
173
174 if(input2 != nullptr)
175 {
176 const int bias_processed_per_iteration_x = num_elems_processed_per_iteration_x;
177
178 const int bias_processed_per_iteration_y = gemm_info.broadcast_bias ? 1 : num_elems_processed_per_iteration_y;
179
180 AccessWindowStatic input2_access(input2, 0, 0,
181 ceil_to_multiple(input2->dimension(0), bias_processed_per_iteration_x),
182 ceil_to_multiple(input2->dimension(1), bias_processed_per_iteration_y));
183
184 window_changed = update_window_and_padding(win, input0_access, input1_access, input2_access) || // window used by the execute_window_loop
185 update_window_and_padding(win_out, output_access); // window used to update the padding requirements of output tensor
186 }
187 else
188 {
189 window_changed = update_window_and_padding(win, input0_access, input1_access) || // window used by the execute_window_loop
190 update_window_and_padding(win_out, output_access); // window used to update the padding requirements of output tensor
191 }
192
193 output_access.set_valid_region(win_out, ValidRegion(Coordinates(0, 0), output->tensor_shape()));
194
195 // Collapse along the Z direction
196 // This collapse needs to be here in order to tune the Z dimension of LWS
197 Window collapsed = win;
198 const unsigned int dimension_to_collapse = std::min(static_cast<unsigned int>(output->num_dimensions()), 2u);
199 collapsed = win.collapse(win, dimension_to_collapse);
200
201 Status err = (window_changed) ? ARM_COMPUTE_CREATE_ERROR(ErrorCode::RUNTIME_ERROR, "Insufficient Padding!") : Status{};
202 return std::make_pair(err, collapsed);
203 }
204 } // namespace
205
CLGEMMMatrixMultiplyReshapedKernel()206 CLGEMMMatrixMultiplyReshapedKernel::CLGEMMMatrixMultiplyReshapedKernel()
207 : _input0(nullptr), _input1(nullptr), _input2(nullptr), _output(nullptr), _slide_matrix_b(true), _reinterpret_output_as_3d(false), _use_dummy_work_items(false), _add_bias(false),
208 _broadcast_bias(false), _export_to_cl_image(false), _k(1)
209 {
210 }
211
configure(const ICLTensor * input0,const ICLTensor * input1,const ICLTensor * input2,ICLTensor * output,float alpha,float beta,const GEMMLHSMatrixInfo & lhs_info,const GEMMRHSMatrixInfo & rhs_info,const GEMMKernelInfo & gemm_info)212 void CLGEMMMatrixMultiplyReshapedKernel::configure(const ICLTensor *input0, const ICLTensor *input1, const ICLTensor *input2, ICLTensor *output, float alpha, float beta,
213 const GEMMLHSMatrixInfo &lhs_info,
214 const GEMMRHSMatrixInfo &rhs_info, const GEMMKernelInfo &gemm_info)
215 {
216 configure(CLKernelLibrary::get().get_compile_context(), input0, input1, input2, output, alpha, beta, lhs_info, rhs_info, gemm_info);
217 }
218
configure(const CLCompileContext & compile_context,const ICLTensor * input0,const ICLTensor * input1,const ICLTensor * input2,ICLTensor * output,float alpha,float beta,const GEMMLHSMatrixInfo & lhs_info,const GEMMRHSMatrixInfo & rhs_info,const GEMMKernelInfo & gemm_info)219 void CLGEMMMatrixMultiplyReshapedKernel::configure(const CLCompileContext &compile_context, const ICLTensor *input0, const ICLTensor *input1, const ICLTensor *input2, ICLTensor *output, float alpha,
220 float beta,
221 const GEMMLHSMatrixInfo &lhs_info,
222 const GEMMRHSMatrixInfo &rhs_info, const GEMMKernelInfo &gemm_info)
223 {
224 ARM_COMPUTE_ERROR_ON_NULLPTR(input0, input1, output);
225
226 ARM_COMPUTE_ERROR_THROW_ON(validate_arguments(input0->info(), input1->info(), (input2 != nullptr ? input2->info() : nullptr), output->info(), alpha, beta, lhs_info, rhs_info, gemm_info));
227
228 auto padding_info = get_padding_info({ input0, output });
229 _input0 = input0;
230 _input1 = input1;
231 _input2 = helpers::float_ops::is_zero(beta) ? nullptr : input2;
232 _output = output;
233 _reinterpret_output_as_3d = gemm_info.depth_output_gemm3d != 0;
234 _use_dummy_work_items = preferred_dummy_work_items_support(CLKernelLibrary::get().get_device());
235 _add_bias = _input2 != nullptr;
236 _broadcast_bias = gemm_info.broadcast_bias;
237 _export_to_cl_image = rhs_info.export_to_cl_image;
238 _k = gemm_info.k;
239
240 // Check if we need to slide the matrix B
241 const unsigned int num_dimensions_input0 = _input0->info()->num_dimensions();
242 _slide_matrix_b = (_input1->info()->num_dimensions() >= num_dimensions_input0);
243
244 ElementsProcessed num_elements_processed{};
245
246 // Configure kernel window
247 auto win_config = validate_and_configure_window(input0->info(), input1->info(), input2 != nullptr ? input2->info() : nullptr, output->info(), lhs_info, rhs_info, gemm_info, num_elements_processed);
248 ARM_COMPUTE_ERROR_THROW_ON(win_config.first);
249 ICLKernel::configure_internal(win_config.second);
250
251 const bool enable_mixed_precision = gemm_info.fp_mixed_precision;
252 const DataType data_type = input0->info()->data_type();
253
254 // Calculate partial (store instead of load) M0 and partial N0 for the partial blocks at the end of a row/column if any. This is to avoid padding.
255 const unsigned int internal_m = _reinterpret_output_as_3d ? gemm_info.m : output->info()->dimension(1);
256
257 const unsigned int partial_store_m0 = internal_m % lhs_info.m0;
258 const unsigned int partial_store_n0 = gemm_info.n % rhs_info.n0;
259
260 // Create build options
261 CLBuildOptions build_opts;
262 build_opts.add_option_if(!(helpers::float_ops::is_one(alpha)), "-DALPHA=" + float_to_string_with_full_precision(alpha));
263 build_opts.add_option_if(_input2 != nullptr, "-DBETA=" + float_to_string_with_full_precision(beta));
264 build_opts.add_option_if(helpers::float_ops::is_one(beta), "-DUNIT_BETA");
265 build_opts.add_option_if(_reinterpret_output_as_3d, "-DREINTERPRET_OUTPUT_AS_3D");
266 build_opts.add_option_if(_reinterpret_output_as_3d, "-DHEIGHT_GEMM3D=" + support::cpp11::to_string(output->info()->dimension(1)));
267 build_opts.add_option_if(_reinterpret_output_as_3d, "-DDEPTH_GEMM3D=" + support::cpp11::to_string(output->info()->dimension(2)));
268 build_opts.add_option_if(gemm_info.broadcast_bias, "-DBROADCAST_BIAS");
269 build_opts.add_option_if(!_slide_matrix_b, "-DMATRIX_B_DEPTH=" + support::cpp11::to_string(input1->info()->dimension(2)));
270 build_opts.add_option_if(lhs_info.interleave, "-DLHS_INTERLEAVE");
271 build_opts.add_option_if(rhs_info.interleave, "-DRHS_INTERLEAVE");
272 build_opts.add_option_if(lhs_info.transpose, "-DLHS_TRANSPOSE");
273 build_opts.add_option_if(_use_dummy_work_items, "-DDUMMY_WORK_ITEMS");
274 build_opts.add_option_if(gemm_info.activation_info.enabled(), "-DACTIVATION_TYPE=" + lower_string(string_from_activation_func(gemm_info.activation_info.activation())));
275 build_opts.add_option_if(gemm_info.activation_info.enabled(), "-DA_VAL=" + float_to_string_with_full_precision(gemm_info.activation_info.a()));
276 build_opts.add_option_if(gemm_info.activation_info.enabled(), "-DB_VAL=" + float_to_string_with_full_precision(gemm_info.activation_info.b()));
277 build_opts.add_option_if(enable_mixed_precision, "-DMIXED_PRECISION");
278 build_opts.add_option_if(rhs_info.export_to_cl_image, "-DOPENCL_IMAGE_SUPPORT");
279 build_opts.add_option("-DRHS_HEIGHT=" + support::cpp11::to_string(input1->info()->dimension(1)));
280 build_opts.add_option("-DDATA_TYPE=" + get_cl_type_from_data_type(data_type));
281 build_opts.add_option("-DDATA_TYPE_ACCUMULATOR=" + (enable_mixed_precision ? get_cl_type_from_data_type(DataType::F32) : get_cl_type_from_data_type(data_type)));
282 build_opts.add_option("-DM=" + support::cpp11::to_string(gemm_info.m));
283 build_opts.add_option("-DN=" + support::cpp11::to_string(gemm_info.n));
284 build_opts.add_option("-DK=" + support::cpp11::to_string(gemm_info.k));
285 build_opts.add_option("-DM0=" + support::cpp11::to_string(lhs_info.m0));
286 build_opts.add_option("-DN0=" + support::cpp11::to_string(rhs_info.n0));
287 build_opts.add_option("-DK0=" + support::cpp11::to_string(lhs_info.k0));
288 build_opts.add_option("-DV0=" + support::cpp11::to_string(lhs_info.v0));
289 build_opts.add_option("-DH0=" + support::cpp11::to_string(rhs_info.h0));
290 build_opts.add_option("-DPARTIAL_STORE_M0=" + support::cpp11::to_string(partial_store_m0));
291 build_opts.add_option("-DPARTIAL_STORE_N0=" + support::cpp11::to_string(partial_store_n0));
292
293 std::string kernel_name("gemm_mm_reshaped_");
294 kernel_name += lhs_info.transpose ? "lhs_t_" : "lhs_nt_";
295 kernel_name += rhs_info.transpose ? "rhs_t" : "rhs_nt";
296 kernel_name += rhs_info.export_to_cl_image ? "_texture" : "";
297
298 // Create kernel
299 _kernel = create_kernel(compile_context, kernel_name, build_opts.options());
300
301 // Set config_id for enabling LWS tuning
302 _config_id = kernel_name;
303 _config_id += "_";
304 _config_id += (_add_bias ? "add_bias_" : "");
305 _config_id += (_broadcast_bias ? "broadcast_bias_" : "");
306 _config_id += (_reinterpret_output_as_3d ? "3do_" : "");
307 _config_id += (gemm_info.activation_info.enabled() ? "fused_activation_" : "");
308 _config_id += lower_string(string_from_data_type(input0->info()->data_type()));
309 _config_id += "_";
310 _config_id += (enable_mixed_precision ? "mixed_precision_" : "");
311 _config_id += support::cpp11::to_string(output->info()->dimension(1));
312 _config_id += "_";
313 _config_id += support::cpp11::to_string(output->info()->dimension(0));
314 _config_id += "_";
315 _config_id += support::cpp11::to_string(gemm_info.k);
316 _config_id += "_";
317 _config_id += support::cpp11::to_string(output->info()->dimension(2));
318 _config_id += "_";
319 _config_id += support::cpp11::to_string(lhs_info.m0);
320 _config_id += "_";
321 _config_id += support::cpp11::to_string(rhs_info.n0);
322 _config_id += "_";
323 _config_id += support::cpp11::to_string(lhs_info.k0);
324 _config_id += "_";
325 _config_id += support::cpp11::to_string(lhs_info.v0);
326 _config_id += "_";
327 _config_id += support::cpp11::to_string(rhs_info.h0);
328 _config_id += "_";
329 _config_id += support::cpp11::to_string(lhs_info.interleave);
330 _config_id += "_";
331 _config_id += support::cpp11::to_string(rhs_info.interleave);
332
333 ARM_COMPUTE_ERROR_ON(has_padding_changed(padding_info));
334 }
335
validate(const ITensorInfo * input0,const ITensorInfo * input1,const ITensorInfo * input2,const ITensorInfo * output,float alpha,float beta,const GEMMLHSMatrixInfo & lhs_info,const GEMMRHSMatrixInfo & rhs_info,const GEMMKernelInfo & gemm_info)336 Status CLGEMMMatrixMultiplyReshapedKernel::validate(const ITensorInfo *input0, const ITensorInfo *input1, const ITensorInfo *input2, const ITensorInfo *output, float alpha, float beta,
337 const GEMMLHSMatrixInfo &lhs_info,
338 const GEMMRHSMatrixInfo &rhs_info, const GEMMKernelInfo &gemm_info)
339 {
340 ElementsProcessed num_elements_processed{};
341 ARM_COMPUTE_RETURN_ON_ERROR(validate_arguments(input0, input1, input2, output, alpha, beta, lhs_info, rhs_info, gemm_info));
342 ARM_COMPUTE_RETURN_ON_ERROR(validate_and_configure_window(input0->clone().get(),
343 input1->clone().get(),
344 input2 != nullptr ? input2->clone().get() : nullptr,
345 output->clone().get(),
346 lhs_info,
347 rhs_info,
348 gemm_info,
349 num_elements_processed)
350 .first);
351
352 return Status{};
353 }
354
run(const Window & window,cl::CommandQueue & queue)355 void CLGEMMMatrixMultiplyReshapedKernel::run(const Window &window, cl::CommandQueue &queue)
356 {
357 ARM_COMPUTE_ERROR_ON_UNCONFIGURED_KERNEL(this);
358 ARM_COMPUTE_ERROR_ON_INVALID_SUBWINDOW(ICLKernel::window(), window);
359
360 if(_input1->info()->num_dimensions() < 3)
361 {
362 // The stride_z for matrix B must be zero if we do not slice
363 ARM_COMPUTE_ERROR_ON(_input1->info()->strides_in_bytes()[3] != 0);
364 }
365
366 Window slice = window.first_slice_window_3D();
367 Window slice_matrix_b = slice;
368
369 slice_matrix_b.set(Window::DimX, Window::Dimension(0, 1, 1));
370 slice_matrix_b.set(Window::DimY, Window::Dimension(0, 1, 1));
371
372 const unsigned int total_cross_plane_pad = _output->info()->padding().top + _output->info()->padding().bottom;
373
374 cl::Image2D input1_image2d;
375
376 if(_export_to_cl_image)
377 {
378 const TensorShape shape2d(_input1->info()->dimension(0) / 4, _input1->info()->dimension(1) * _input1->info()->dimension(2));
379 const size_t image_row_pitch = _input1->info()->strides_in_bytes()[1];
380
381 input1_image2d = create_image2d_from_buffer(CLKernelLibrary::get().context(), _input1->cl_buffer(), shape2d, _input1->info()->data_type(), image_row_pitch);
382 }
383
384 do
385 {
386 Window slice_b = slice;
387 // Don't slice matrix B along the z dimension if matrix B has just 2 dimensions and matrix A more than 2
388 // This scenario can happen when the matrix multiplication is used to perform a convolution operation
389 if(!_slide_matrix_b)
390 {
391 slice_b = slice_matrix_b;
392 }
393
394 unsigned int idx = 0;
395
396 // LHS buffer
397 add_2D_tensor_argument(idx, _input0, slice);
398
399 // RHS buffer or RHS OpenCL image (_export_to_cl_image == true)
400 if(_export_to_cl_image)
401 {
402 _kernel.setArg(idx++, input1_image2d);
403 }
404 else
405 {
406 add_2D_tensor_argument(idx, _input1, slice_b);
407 }
408
409 // Bias buffer (_add_bias == true)
410 add_2D_tensor_argument_if(_add_bias, idx, _input2, slice);
411
412 // Output buffer
413 add_2D_tensor_argument(idx, _output, slice);
414
415 // K dimension (not used if _export_to_cl_image == true)
416 _kernel.setArg<cl_uint>(idx++, static_cast<unsigned int>(_k));
417
418 // LHS stride_z
419 _kernel.setArg<cl_uint>(idx++, static_cast<unsigned int>(_input0->info()->strides_in_bytes()[2]));
420
421 // RHS stride_z (not used if _export_to_cl_image == true)
422 _kernel.setArg<cl_uint>(idx++, static_cast<unsigned int>(_input1->info()->strides_in_bytes()[2]));
423
424 // Bias stride_z (if _add_bias == true)
425 if(_add_bias)
426 {
427 _kernel.setArg<cl_uint>(idx++, static_cast<unsigned int>(_input2->info()->strides_in_bytes()[2]));
428 }
429
430 // Output stride_z
431 _kernel.setArg<cl_uint>(idx++, static_cast<unsigned int>(_output->info()->strides_in_bytes()[2]));
432
433 // Cross-plan padding (if _reinterpret_output_as_3d = true)
434 if(_reinterpret_output_as_3d)
435 {
436 _kernel.setArg<cl_uint>(idx++, static_cast<unsigned int>(total_cross_plane_pad));
437 }
438
439 // Dispatch kernel
440 enqueue(queue, *this, slice, lws_hint(), _use_dummy_work_items);
441 }
442 while(window.slide_window_slice_3D(slice));
443 }
444