1 /*
2 * Copyright (c) 2017-2018 Arm Limited.
3 *
4 * SPDX-License-Identifier: MIT
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to
8 * deal in the Software without restriction, including without limitation the
9 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
10 * sell copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in all
14 * copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 * SOFTWARE.
23 */
24
25 #ifndef ARM_COMPUTE_HELPER_CS_H
26 #define ARM_COMPUTE_HELPER_CS_H
27
28 #define SHADER_PARAMS_DECLARATION \
29 layout(std140, binding = 0) uniform shader_params
30
31 #define TENSOR_DECLARATION(location, buffer_type, type, ptr_name, shift_name, element_shift, access) \
32 layout(std430, binding = location) access buffer buffer_type \
33 { \
34 type ptr_name[]; \
35 }; \
36 const uint shift_name = uint(element_shift)
37
38 struct VectorAttributes
39 {
40 uint stride_x; /**< Stride of the vector in X dimension (in bytes) */
41 uint step_x; /**< stride_x * number of elements along X processed per workitem (in bytes) */
42 uint offset_first_element_in_bytes; /**< The offset of the first element in the vector (in bytes) */
43 uint padding; /**< The padding to rounding up the structure to a multiple of a vec4 */
44 };
45
46 struct ImageAttributes
47 {
48 uint stride_x; /**< Stride of the image in X dimension (in bytes) */
49 uint step_x; /**< stride_x * number of elements along X processed per workitem (in bytes) */
50 uint stride_y; /**< Stride of the image in Y dimension (in bytes) */
51 uint step_y; /**< stride_y * number of elements along Y processed per workitem (in bytes) */
52 uint offset_first_element_in_bytes; /**< The offset of the first element in the image (in bytes) */
53 uint padding1; /**< The padding to rounding up the structure to a multiple of a vec4 */
54 uint padding2; /**< The padding to rounding up the structure to a multiple of a vec4 */
55 uint padding3; /**< The padding to rounding up the structure to a multiple of a vec4 */
56 };
57
58 struct Tensor3DAttributes
59 {
60 uint stride_x; /**< Stride of the tensor in X dimension (in bytes) */
61 uint step_x; /**< stride_x * number of elements along X processed per workitem (in bytes) */
62 uint stride_y; /**< Stride of the tensor in Y dimension (in bytes) */
63 uint step_y; /**< stride_y * number of elements along Y processed per workitem (in bytes) */
64 uint stride_z; /**< Stride of the tensor in Z dimension (in bytes) */
65 uint step_z; /**< stride_z * number of elements along Z processed per workitem (in bytes) */
66 uint offset_first_element_in_bytes; /**< The offset of the first element in the tensor (in bytes) */
67 uint padding; /**< The padding to rounding up the structure to a multiple of a vec4 */
68 };
69
70 struct VectorIterator
71 {
72 int current_offset_in_bytes; /**< Current offset of vector (in bytes) */
73 int stride_x; /**< Stride of the vector in X dimension (in bytes) */
74 int element_shift; /**< The number of bits to shift by for one element */
75 };
76
77 struct ImageIterator
78 {
79 int current_offset_in_bytes; /**< Current offset of image (in bytes) */
80 int stride_x; /**< Stride of the image in X dimension (in bytes) */
81 int stride_y; /**< Stride of the image in Y dimension (in bytes) */
82 int element_shift; /**< The number of bits to shift by for one element */
83 };
84
85 struct Tensor3DIterator
86 {
87 int current_offset_in_bytes; /**< Current offset of tensor (in bytes) */
88 int stride_x; /**< Stride of the tensor in X dimension (in bytes) */
89 int stride_y; /**< Stride of the tensor in Y dimension (in bytes) */
90 int stride_z; /**< Stride of the tensor in Z dimension (in bytes) */
91 int element_shift; /**< The number of bits to shift by for one element */
92 };
93
94 #define CONVERT_TO_VECTOR_ITERATOR(attrs, element_shift) \
95 update_vector_iter_offset(element_shift, attrs.offset_first_element_in_bytes, \
96 attrs.stride_x, attrs.step_x)
97
98 #define CONVERT_TO_VECTOR_ITERATOR_NO_STEP(attrs, element_shift) \
99 update_vector_iter_offset(element_shift, attrs.offset_first_element_in_bytes, \
100 attrs.stride_x, uint(0))
101
102 #define CONVERT_TO_IMAGE_ITERATOR(attrs, element_shift) \
103 update_image_iter_offset(element_shift, attrs.offset_first_element_in_bytes, \
104 attrs.stride_x, attrs.step_x, attrs.stride_y, attrs.step_y)
105
106 #define CONVERT_TO_IMAGE_ITERATOR_NO_STEP(attrs, element_shift) \
107 update_image_iter_offset(element_shift, attrs.offset_first_element_in_bytes, \
108 attrs.stride_x, uint(0), attrs.stride_y, uint(0))
109
110 #define CONVERT_TO_TENSOR3D_ITERATOR(attrs, element_shift) \
111 update_tensor3D_iter_offset(element_shift, attrs.offset_first_element_in_bytes, \
112 attrs.stride_x, attrs.step_x, attrs.stride_y, attrs.step_y, attrs.stride_z, attrs.step_z)
113
114 #define CONVERT_TO_TENSOR3D_ITERATOR_NO_STEP(attrs, element_shift) \
115 update_tensor3D_iter_offset(element_shift, attrs.offset_first_element_in_bytes, \
116 attrs.stride_x, uint(0), attrs.stride_y, uint(0), attrs.stride_z, uint(0))
117
118 #define CONVERT_TENSOR3D_TO_IMAGE_ITERATOR(attrs, element_shift) \
119 update_image_from_tensor3D_iter_offset(element_shift, attrs.offset_first_element_in_bytes, \
120 attrs.stride_x, attrs.step_x, attrs.stride_y, attrs.step_y, attrs.stride_z, attrs.step_z)
121
122 #define CONVERT_TENSOR3D_TO_IMAGE_ITERATOR_NO_STEP(attrs, element_shift) \
123 update_image_from_tensor3D_iter_offset(element_shift, attrs.offset_first_element_in_bytes, \
124 attrs.stride_x, uint(0), attrs.stride_y, uint(0), attrs.stride_z, attrs.step_z)
125
126 /** Wrap vector information into a VectorIterator structure, and make the offset to be this workitem's position.
127 *
128 * @param[in] element_shift The number of bits to shift by for one element
129 * @param[in] offset_first_element_in_bytes The offset of the first element in the source vector
130 * @param[in] stride_x Stride of the vector in X dimension (in bytes)
131 * @param[in] step_x stride_x * number of elements along X processed per workitem (in bytes)
132 *
133 * @return A VectorIterator object
134 */
update_vector_iter_offset(uint element_shift,uint offset_first_element_in_bytes,uint stride_x,uint step_x)135 VectorIterator update_vector_iter_offset(uint element_shift, uint offset_first_element_in_bytes, uint stride_x, uint step_x)
136 {
137 VectorIterator vector_iter;
138 vector_iter.element_shift = int(element_shift);
139 vector_iter.stride_x = int(stride_x);
140 vector_iter.current_offset_in_bytes = int(offset_first_element_in_bytes + gl_GlobalInvocationID.x * step_x);
141
142 return vector_iter;
143 }
144
145 /** Wrap image information into an ImageIterator structure, and make the offset to be this workitem's position.
146 *
147 * @param[in] element_shift The number of bits to shift by for one element
148 * @param[in] offset_first_element_in_bytes The offset of the first element in the source image
149 * @param[in] stride_x Stride of the image in X dimension (in bytes)
150 * @param[in] step_x stride_x * number of elements along X processed per workitem (in bytes)
151 * @param[in] stride_y Stride of the image in Y dimension (in bytes)
152 * @param[in] step_y stride_y * number of elements along Y processed per workitem (in bytes)
153 *
154 * @return An ImageIterator object
155 */
update_image_iter_offset(uint element_shift,uint offset_first_element_in_bytes,uint stride_x,uint step_x,uint stride_y,uint step_y)156 ImageIterator update_image_iter_offset(uint element_shift, uint offset_first_element_in_bytes, uint stride_x, uint step_x, uint stride_y, uint step_y)
157 {
158 ImageIterator image_iter;
159 image_iter.element_shift = int(element_shift);
160 image_iter.stride_x = int(stride_x);
161 image_iter.stride_y = int(stride_y);
162 image_iter.current_offset_in_bytes = int(offset_first_element_in_bytes + gl_GlobalInvocationID.x * step_x + gl_GlobalInvocationID.y * step_y);
163
164 return image_iter;
165 }
166
167 /** Wrap 3D tensor information into a Tensor3DIterator structure, and make the offset to be this workitem's position.
168 *
169 * @param[in] element_shift The number of bits to shift by for one element
170 * @param[in] offset_first_element_in_bytes The offset of the first element in the source tersor
171 * @param[in] stride_x Stride of the tersor in X dimension (in bytes)
172 * @param[in] step_x stride_x * number of elements along X processed per workitem (in bytes)
173 * @param[in] stride_y Stride of the tersor in Y dimension (in bytes)
174 * @param[in] step_y stride_y * number of elements along Y processed per workitem (in bytes)
175 * @param[in] stride_z Stride of the tersor in Z dimension (in bytes)
176 * @param[in] step_z stride_z * number of elements along Z processed per workitem (in bytes)
177 *
178 * @return A 3D Tensor3DIterator object
179 */
update_tensor3D_iter_offset(uint element_shift,uint offset_first_element_in_bytes,uint stride_x,uint step_x,uint stride_y,uint step_y,uint stride_z,uint step_z)180 Tensor3DIterator update_tensor3D_iter_offset(uint element_shift, uint offset_first_element_in_bytes, uint stride_x, uint step_x, uint stride_y, uint step_y, uint stride_z, uint step_z)
181 {
182 Tensor3DIterator tensor_iter;
183 tensor_iter.element_shift = int(element_shift);
184 tensor_iter.stride_x = int(stride_x);
185 tensor_iter.stride_y = int(stride_y);
186 tensor_iter.stride_z = int(stride_z);
187 tensor_iter.current_offset_in_bytes = int(offset_first_element_in_bytes + gl_GlobalInvocationID.x * step_x + gl_GlobalInvocationID.y * step_y + gl_GlobalInvocationID.z * step_z);
188
189 return tensor_iter;
190 }
191
192 /** Wrap 3D tensor information into an ImageIterator structure, and make the offset to be this workitem's position.
193 *
194 * @param[in] element_shift The number of bits to shift by for one element
195 * @param[in] offset_first_element_in_bytes The offset of the first element in the source tensor
196 * @param[in] stride_x Stride of the tensor in X dimension (in bytes)
197 * @param[in] step_x stride_x * number of elements along X processed per workitem (in bytes)
198 * @param[in] stride_y Stride of the tensor in Y dimension (in bytes)
199 * @param[in] step_y stride_y * number of elements along Y processed per workitem (in bytes)
200 * @param[in] stride_z Stride of the tensor in Z dimension (in bytes)
201 * @param[in] step_z stride_z * number of elements along Z processed per workitem (in bytes)
202 *
203 * @return An ImageIterator object
204 */
update_image_from_tensor3D_iter_offset(uint element_shift,uint offset_first_element_in_bytes,uint stride_x,uint step_x,uint stride_y,uint step_y,uint stride_z,uint step_z)205 ImageIterator update_image_from_tensor3D_iter_offset(uint element_shift, uint offset_first_element_in_bytes, uint stride_x, uint step_x, uint stride_y, uint step_y, uint stride_z, uint step_z)
206 {
207 ImageIterator image_iter;
208 image_iter.element_shift = int(element_shift);
209 image_iter.stride_x = int(stride_x);
210 image_iter.stride_y = int(stride_y);
211 image_iter.current_offset_in_bytes = int(offset_first_element_in_bytes + gl_GlobalInvocationID.x * step_x + gl_GlobalInvocationID.y * step_y + gl_GlobalInvocationID.z * step_z);
212
213 return image_iter;
214 }
215
216 #define VECTOR_OFFSET(tensor_iter, x) \
217 uint(vector_offset_in_bytes(tensor_iter, int(x)) >> tensor_iter.element_shift)
218
219 #define IMAGE_OFFSET(tensor_iter, x, y) \
220 uint(image_offset_in_bytes(tensor_iter, int(x), int(y)) >> tensor_iter.element_shift)
221
222 #define TENSOR3D_OFFSET(tensor_iter, x, y, z) \
223 uint(tensor3D_offset_in_bytes(tensor_iter, int(x), int(y), int(z)) >> tensor_iter.element_shift)
224
225 #define TENSOR_OFFSET_ADVANCE(tensor_iter, n) \
226 uint((tensor_iter.current_offset_in_bytes >> tensor_iter.element_shift) + int(n))
227
228 #define TENSOR_OFFSET_ADVANCE_IN_BYTES(tensor_iter, n) \
229 uint((tensor_iter.current_offset_in_bytes + int(n)) >> tensor_iter.element_shift)
230
231 #define CURRENT_ITEM_OFFSET(tensor_iter) \
232 uint(tensor_iter.current_offset_in_bytes >> tensor_iter.element_shift)
233
234 #define CURRENT_ITEM_OFFSET_IN_BYTES(tensor_iter) \
235 uint(tensor_iter.current_offset_in_bytes)
236
237 #define TENSOR_ITERATOR_ADVANCE(tensor_iter, n) \
238 tensor_iter.current_offset_in_bytes += (int(n) << tensor_iter.element_shift)
239
240 #define TENSOR_ITERATOR_ADVANCE_IN_BYTES(tensor_iter, n) \
241 tensor_iter.current_offset_in_bytes += int(n)
242
243 #define SET_TENSOR_ITERATOR_OFFSET_IN_BYTES(tensor_iter, n) \
244 tensor_iter.current_offset_in_bytes = int(n)
245
246 /** Get the offset of a VectorIterator
247 *
248 * @param[in] vector_iter The VectorIterator object pointed to the starting position of the buffer
249 * @param[in] x Relative X position
250 *
251 * @return The relative offset of the VectorIterator object (in bytes)
252 */
vector_offset_in_bytes(VectorIterator vector_iter,int x)253 uint vector_offset_in_bytes(VectorIterator vector_iter, int x)
254 {
255 return uint(vector_iter.current_offset_in_bytes + x * vector_iter.stride_x);
256 }
257
258 /** Get the offset of an ImageIterator
259 *
260 * @param[in] vector_iter The ImageIterator object pointed to the starting position of the buffer
261 * @param[in] x Relative X position
262 * @param[in] y Relative Y position
263 *
264 * @return The relative offset of the ImageIterator object (in bytes)
265 */
image_offset_in_bytes(ImageIterator image_iter,int x,int y)266 uint image_offset_in_bytes(ImageIterator image_iter, int x, int y)
267 {
268 return uint(image_iter.current_offset_in_bytes + x * image_iter.stride_x + y * image_iter.stride_y);
269 }
270
271 /** Get the offset of a Tensor3DIterator
272 *
273 * @param[in] vector_iter The Tensor3DIterator object pointed to the starting position of the buffer
274 * @param[in] x Relative X position
275 * @param[in] y Relative Y position
276 * @param[in] z Relative Z position
277 *
278 * @return The relative offset of the Tensor3DIterator object (in bytes)
279 */
tensor3D_offset_in_bytes(Tensor3DIterator tensor_iter,int x,int y,int z)280 uint tensor3D_offset_in_bytes(Tensor3DIterator tensor_iter, int x, int y, int z)
281 {
282 return uint(tensor_iter.current_offset_in_bytes + x * tensor_iter.stride_x + y * tensor_iter.stride_y + z * tensor_iter.stride_z);
283 }
284
285 #define LOAD(tensor_ptr, offset) tensor_ptr[offset]
286 #define STORE(tensor_ptr, offset, data) tensor_ptr[offset] = data
287 #define LOAD_CURRENT_ITEM(tensor_ptr, tensor_iter) tensor_ptr[CURRENT_ITEM_OFFSET(tensor_iter)]
288 #define STORE_CURRENT_ITEM(tensor_ptr, tensor_iter, data) tensor_ptr[CURRENT_ITEM_OFFSET(tensor_iter)] = data
289
290 #define VLOAD2(return_type, tensor_ptr, offset) \
291 return_type(LOAD(tensor_ptr, offset), \
292 LOAD(tensor_ptr, (offset) + uint(1)))
293
294 #define VSTORE2(tensor_ptr, offset, data) \
295 STORE(tensor_ptr, offset, data[0]); \
296 STORE(tensor_ptr, (offset) + uint(1), data[1])
297
298 #define VLOAD2_CURRENT_ITEM(return_type, tensor_ptr, tensor_iter) VLOAD2(return_type, tensor_ptr, CURRENT_ITEM_OFFSET(tensor_iter))
299 #define VSTORE2_CURRENT_ITEM(tensor_ptr, tensor_iter, data) VSTORE2(tensor_ptr, CURRENT_ITEM_OFFSET(tensor_iter), data)
300
301 #define VLOAD3(return_type, tensor_ptr, offset) \
302 return_type(LOAD(tensor_ptr, offset), \
303 LOAD(tensor_ptr, (offset) + uint(1)), \
304 LOAD(tensor_ptr, (offset) + uint(2)))
305
306 #define VSTORE3(tensor_ptr, offset, data) \
307 STORE(tensor_ptr, offset, data[0]); \
308 STORE(tensor_ptr, (offset) + uint(1), data[1]); \
309 STORE(tensor_ptr, (offset) + uint(2), data[2])
310
311 #define VLOAD3_CURRENT_ITEM(return_type, tensor_ptr, tensor_iter) VLOAD3(return_type, tensor_ptr, CURRENT_ITEM_OFFSET(tensor_iter))
312 #define VSTORE3_CURRENT_ITEM(tensor_ptr, tensor_iter, data) VSTORE3(tensor_ptr, CURRENT_ITEM_OFFSET(tensor_iter), data)
313
314 #define VLOAD4(return_type, tensor_ptr, offset) \
315 return_type(LOAD(tensor_ptr, offset), \
316 LOAD(tensor_ptr, (offset) + uint(1)), \
317 LOAD(tensor_ptr, (offset) + uint(2)), \
318 LOAD(tensor_ptr, (offset) + uint(3)))
319
320 #define VSTORE4(tensor_ptr, offset, data) \
321 STORE(tensor_ptr, offset, data[0]); \
322 STORE(tensor_ptr, (offset) + uint(1), data[1]); \
323 STORE(tensor_ptr, (offset) + uint(2), data[2]); \
324 STORE(tensor_ptr, (offset) + uint(3), data[3])
325
326 #define VLOAD4_CURRENT_ITEM(return_type, tensor_ptr, tensor_iter) VLOAD4(return_type, tensor_ptr, CURRENT_ITEM_OFFSET(tensor_iter))
327 #define VSTORE4_CURRENT_ITEM(tensor_ptr, tensor_iter, data) VSTORE4(tensor_ptr, CURRENT_ITEM_OFFSET(tensor_iter), data)
328
329 #define VLOAD5(return_type, tensor_ptr, offset) \
330 return_type(LOAD(tensor_ptr, offset), \
331 LOAD(tensor_ptr, (offset) + uint(1)), \
332 LOAD(tensor_ptr, (offset) + uint(2)), \
333 LOAD(tensor_ptr, (offset) + uint(3)), \
334 LOAD(tensor_ptr, (offset) + uint(4)))
335
336 #define VSTORE5(tensor_ptr, offset, data) \
337 STORE(tensor_ptr, offset, data[0]); \
338 STORE(tensor_ptr, (offset) + uint(1), data[1]); \
339 STORE(tensor_ptr, (offset) + uint(2), data[2]); \
340 STORE(tensor_ptr, (offset) + uint(3), data[3]); \
341 STORE(tensor_ptr, (offset) + uint(4), data[4])
342
343 #define VLOAD5_CURRENT_ITEM(return_type, tensor_ptr, tensor_iter) VLOAD5(return_type, tensor_ptr, CURRENT_ITEM_OFFSET(tensor_iter))
344 #define VSTORE5_CURRENT_ITEM(tensor_ptr, tensor_iter, data) VSTORE5(tensor_ptr, CURRENT_ITEM_OFFSET(tensor_iter), data)
345
346 /** Converting the vec4 object to 4 half-precision (16-bits) floating point values and packing into a uvec2 object
347 *
348 * @param[in] data The vec4 object to be packed
349 *
350 * @return The packed uvec2 object
351 */
pack4_half(mediump vec4 data)352 highp uvec2 pack4_half(mediump vec4 data)
353 {
354 return uvec2(packHalf2x16(data.xy), packHalf2x16(data.zw));
355 }
356
357 /** Unpacking the uvec2 object to 4 half-precision (16-bits) floating point values and converting to a vec4 object
358 *
359 * @param[in] packed_data The uvec2 object to be unpacked
360 *
361 * @return The unpacked vec4 object
362 */
unpack4_half(highp uvec2 packed_data)363 mediump vec4 unpack4_half(highp uvec2 packed_data)
364 {
365 return vec4(unpackHalf2x16(packed_data.x), unpackHalf2x16(packed_data.y));
366 }
367
368 /** Unpacking the uvec3 object to 6 half-precision (16-bits) floating point values and converting to a vec2[3] object
369 *
370 * @param[in] packed_data The uvec3 object to be unpacked
371 *
372 * @return The unpacked vec2[3] object
373 */
unpack6_half(highp uvec3 packed_data)374 mediump vec2[3] unpack6_half(highp uvec3 packed_data)
375 {
376 return vec2[3](unpackHalf2x16(packed_data[0]),
377 unpackHalf2x16(packed_data[1]),
378 unpackHalf2x16(packed_data[2]));
379 }
380
381 /** Converting the vec4[2] object to 8 half-precision (16-bits) floating point values and packing into a uvec4 object
382 *
383 * @param[in] data The vec4[2] object to be packed
384 *
385 * @return The packed uvec4 object
386 */
pack8_half(mediump vec4 data[2])387 highp uvec4 pack8_half(mediump vec4 data[2])
388 {
389 return uvec4(packHalf2x16(data[0].xy), packHalf2x16(data[0].zw),
390 packHalf2x16(data[1].xy), packHalf2x16(data[1].zw));
391 }
392
393 /** Unpacking the uvec4 object to 8 half-precision (16-bits) floating point values and converting to a vec4[2] object
394 *
395 * @param[in] packed_data The uvec4 object to be unpacked
396 *
397 * @return The unpacked vec4[2] object
398 */
unpack8_half(highp uvec4 packed_data)399 mediump vec4[2] unpack8_half(highp uvec4 packed_data)
400 {
401 return vec4[2](vec4(unpackHalf2x16(packed_data.x), unpackHalf2x16(packed_data.y)),
402 vec4(unpackHalf2x16(packed_data.z), unpackHalf2x16(packed_data.w)));
403 }
404
405 /** Unpacking the uvec2[3] object to 12 half-precision (16-bits) floating point values and converting to a vec4[3] object
406 *
407 * @param[in] packed_data The uvec2[3] object to be unpacked
408 *
409 * @return The unpacked vec4[3] object
410 */
unpack12_half(highp uvec2[3]packed_data)411 mediump vec4[3] unpack12_half(highp uvec2[3] packed_data)
412 {
413 return vec4[3](vec4(unpackHalf2x16(packed_data[0].x), unpackHalf2x16(packed_data[0].y)),
414 vec4(unpackHalf2x16(packed_data[1].x), unpackHalf2x16(packed_data[1].y)),
415 vec4(unpackHalf2x16(packed_data[2].x), unpackHalf2x16(packed_data[2].y)));
416 }
417
418 // For half-precision (16-bits) floating point packed into a "uint" element
419 #define LOAD_UNPACK2_HALF(tensor_ptr, offset) unpackHalf2x16(uint(LOAD(tensor_ptr, offset)))
420 #define STORE_PACK2_HALF(tensor_ptr, offset, data) STORE(tensor_ptr, offset, packHalf2x16(data))
421 #define LOAD_UNPACK2_CURRENT_ITEM_HALF(tensor_ptr, tensor_iter) LOAD_UNPACK2_HALF(tensor_ptr, CURRENT_ITEM_OFFSET(tensor_iter))
422 #define STORE_PACK2_CURRENT_ITEM_HALF(tensor_ptr, tensor_iter, data) STORE_PACK2_HALF(tensor_ptr, CURRENT_ITEM_OFFSET(tensor_iter), data)
423
424 #define VLOAD2_UNPACK4_HALF(tensor_ptr, offset) unpack4_half(VLOAD2(uvec2, tensor_ptr, offset))
425 #define VSTORE2_PACK4_HALF(tensor_ptr, offset, data) VSTORE2(tensor_ptr, offset, pack4_half(data))
426 #define VLOAD2_UNPACK4_CURRENT_ITEM_HALF(tensor_ptr, tensor_iter) VLOAD2_UNPACK4_HALF(tensor_ptr, CURRENT_ITEM_OFFSET(tensor_iter))
427 #define VSTORE2_PACK4_CURRENT_ITEM_HALF(tensor_ptr, tensor_iter, data) VSTORE2_PACK4_HALF(tensor_ptr, CURRENT_ITEM_OFFSET(tensor_iter), data)
428
429 #define VLOAD3_UNPACK6_HALF(tensor_ptr, offset) unpack6_half(VLOAD3(uvec3, tensor_ptr, offset))
430 #define VLOAD3_UNPACK6_CURRENT_ITEM_HALF(tensor_ptr, tensor_iter) VLOAD3_UNPACK6_HALF(tensor_ptr, CURRENT_ITEM_OFFSET(tensor_iter))
431
432 #define VLOAD4_UNPACK8_HALF(tensor_ptr, offset) unpack8_half(VLOAD4(uvec4, tensor_ptr, offset))
433 #define VSTORE4_PACK8_HALF(tensor_ptr, offset, data) VSTORE4(tensor_ptr, offset, pack8_half(data))
434 #define VLOAD4_UNPACK8_CURRENT_ITEM_HALF(tensor_ptr, tensor_iter) VLOAD4_UNPACK8_HALF(tensor_ptr, CURRENT_ITEM_OFFSET(tensor_iter))
435 #define VSTORE4_PACK8_CURRENT_ITEM_HALF(tensor_ptr, tensor_iter, data) VSTORE4_PACK8_HALF(tensor_ptr, CURRENT_ITEM_OFFSET(tensor_iter), data)
436
437 // For half-precision (16-bits) floating point packed into a "uvec2" element
438 #define LOAD_UNPACK4_HALF(tensor_ptr, offset) unpack4_half(uvec2(LOAD(tensor_ptr, offset)))
439 #define STORE_PACK4_HALF(tensor_ptr, offset, data) STORE(tensor_ptr, offset, pack4_half(data))
440 #define LOAD_UNPACK4_CURRENT_ITEM_HALF(tensor_ptr, tensor_iter) LOAD_UNPACK4_HALF(tensor_ptr, CURRENT_ITEM_OFFSET(tensor_iter))
441 #define STORE_PACK4_CURRENT_ITEM_HALF(tensor_ptr, tensor_iter, data) STORE_PACK4_HALF(tensor_ptr, CURRENT_ITEM_OFFSET(tensor_iter), data)
442
443 #define VLOAD2_UNPACK8_HALF(tensor_ptr, offset) unpack8_half(VLOAD2(uvec4, tensor_ptr, offset))
444 #define VSTORE2_PACK8_HALF(tensor_ptr, offset, data) VSTORE2(tensor_ptr, offset, pack8_half(data))
445 #define VLOAD2_UNPACK8_CURRENT_ITEM_HALF(tensor_ptr, tensor_iter) VLOAD2_UNPACK8_HALF(tensor_ptr, CURRENT_ITEM_OFFSET(tensor_iter))
446 #define VSTORE2_PACK8_CURRENT_ITEM_HALF(tensor_ptr, tensor_iter, data) VSTORE2_PACK8_HALF(tensor_ptr, CURRENT_ITEM_OFFSET(tensor_iter), data)
447
448 #define VLOAD3_UNPACK12_HALF(tensor_ptr, offset) unpack12_half(VLOAD3(uvec2[3], tensor_ptr, offset))
449 #define VLOAD3_UNPACK12_CURRENT_ITEM_HALF(tensor_ptr, tensor_iter) VLOAD3_UNPACK12_HALF(tensor_ptr, CURRENT_ITEM_OFFSET(tensor_iter))
450
451 // For half-precision (16-bits) floating point packed into a "uvec4" element
452 #define LOAD_UNPACK8_HALF(tensor_ptr, offset) unpack8_half(uvec4(LOAD(tensor_ptr, offset)))
453 #define STORE_PACK8_HALF(tensor_ptr, offset, data) STORE(tensor_ptr, offset, pack8_half(data))
454 #define LOAD_UNPACK8_CURRENT_ITEM_HALF(tensor_ptr, tensor_iter) LOAD_UNPACK8_HALF(tensor_ptr, CURRENT_ITEM_OFFSET(tensor_iter))
455 #define STORE_PACK8_CURRENT_ITEM_HALF(tensor_ptr, tensor_iter, data) STORE_PACK8_HALF(tensor_ptr, CURRENT_ITEM_OFFSET(tensor_iter), data)
456
457 /** Converting the uvec4 object to 4 low-precision uint values and packing into a uint object
458 *
459 * @param[in] data The uvec4 object to be packed
460 *
461 * @return The packed uint object
462 */
pack4_u8(lowp uvec4 data)463 highp uint pack4_u8(lowp uvec4 data)
464 {
465 highp uint r = uint(0);
466
467 for(int i = 0; i < 4; i++)
468 {
469 r |= data[i] << uint(i * 8);
470 }
471
472 return r;
473 }
474
475 /** Unpacking the uint object to 4 low-precision uint values and converting to a uvec4 object
476 *
477 * @param[in] packed_data The uint object to be unpacked
478 *
479 * @return The unpacked uvec4 object
480 */
unpack4_u8(highp uint packed_data)481 lowp uvec4 unpack4_u8(highp uint packed_data)
482 {
483 lowp uvec4 uvec;
484
485 for(int i = 0; i < 4; i++)
486 {
487 uvec[i] = (packed_data >> uint(i * 8)) & uint(0xFF);
488 }
489
490 return uvec;
491 }
492
493 #define LOAD_UNPACK4_U8(tensor_ptr, offset) unpack4_u8(uint(LOAD(tensor_ptr, offset)))
494 #define STORE_PACK4_U8(tensor_ptr, offset, data) STORE(tensor_ptr, offset, pack4_u8(data))
495 #define LOAD_UNPACK4_CURRENT_ITEM_U8(tensor_ptr, tensor_iter) LOAD_UNPACK4_U8(tensor_ptr, CURRENT_ITEM_OFFSET(tensor_iter))
496 #define STORE_PACK4_CURRENT_ITEM_U8(tensor_ptr, tensor_iter, data) STORE_PACK4_U8(tensor_ptr, CURRENT_ITEM_OFFSET(tensor_iter), data)
497
498 #endif // ARM_COMPUTE_HELPER_CS_H
499