1 //
2 // Copyright 2012 Francisco Jerez
3 //
4 // Permission is hereby granted, free of charge, to any person obtaining a
5 // copy of this software and associated documentation files (the "Software"),
6 // to deal in the Software without restriction, including without limitation
7 // the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 // and/or sell copies of the Software, and to permit persons to whom the
9 // Software is furnished to do so, subject to the following conditions:
10 //
11 // The above copyright notice and this permission notice shall be included in
12 // all copies or substantial portions of the Software.
13 //
14 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 // THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 // OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 // ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 // OTHER DEALINGS IN THE SOFTWARE.
21 //
22
23 #include "util/format/u_format.h"
24 #include "util/u_math.h"
25 #include "api/util.hpp"
26 #include "core/memory.hpp"
27 #include "core/format.hpp"
28
29 using namespace clover;
30
31 namespace {
32 cl_mem_flags
validate_flags(cl_mem d_parent,cl_mem_flags d_flags,bool svm)33 validate_flags(cl_mem d_parent, cl_mem_flags d_flags, bool svm) {
34 const cl_mem_flags dev_access_flags =
35 CL_MEM_READ_WRITE | CL_MEM_WRITE_ONLY | CL_MEM_READ_ONLY;
36 const cl_mem_flags host_ptr_flags =
37 CL_MEM_USE_HOST_PTR | CL_MEM_ALLOC_HOST_PTR | CL_MEM_COPY_HOST_PTR;
38 const cl_mem_flags host_access_flags =
39 CL_MEM_HOST_WRITE_ONLY | CL_MEM_HOST_READ_ONLY | CL_MEM_HOST_NO_ACCESS;
40 const cl_mem_flags svm_flags =
41 CL_MEM_SVM_FINE_GRAIN_BUFFER | CL_MEM_SVM_ATOMICS;
42
43 const cl_mem_flags valid_flags =
44 dev_access_flags
45 | (svm || d_parent ? 0 : host_ptr_flags)
46 | (svm ? svm_flags : host_access_flags);
47
48 if ((d_flags & ~valid_flags) ||
49 util_bitcount(d_flags & dev_access_flags) > 1 ||
50 util_bitcount(d_flags & host_access_flags) > 1)
51 throw error(CL_INVALID_VALUE);
52
53 if ((d_flags & CL_MEM_USE_HOST_PTR) &&
54 (d_flags & (CL_MEM_COPY_HOST_PTR | CL_MEM_ALLOC_HOST_PTR)))
55 throw error(CL_INVALID_VALUE);
56
57 if ((d_flags & CL_MEM_SVM_ATOMICS) &&
58 !(d_flags & CL_MEM_SVM_FINE_GRAIN_BUFFER))
59 throw error(CL_INVALID_VALUE);
60
61 if (d_parent) {
62 const auto &parent = obj(d_parent);
63 const cl_mem_flags flags = (d_flags |
64 (d_flags & dev_access_flags ? 0 :
65 parent.flags() & dev_access_flags) |
66 (d_flags & host_access_flags ? 0 :
67 parent.flags() & host_access_flags) |
68 (parent.flags() & host_ptr_flags));
69
70 if (~flags & parent.flags() & (dev_access_flags & ~CL_MEM_READ_WRITE))
71 throw error(CL_INVALID_VALUE);
72
73 // Check if new host access flags cause a mismatch between
74 // host-read/write-only.
75 if (!(flags & CL_MEM_HOST_NO_ACCESS) &&
76 (~flags & parent.flags() & host_access_flags))
77 throw error(CL_INVALID_VALUE);
78
79 return flags;
80
81 } else {
82 return d_flags | (d_flags & dev_access_flags ? 0 : CL_MEM_READ_WRITE);
83 }
84 }
85 }
86
87 CLOVER_API cl_mem
clCreateBuffer(cl_context d_ctx,cl_mem_flags d_flags,size_t size,void * host_ptr,cl_int * r_errcode)88 clCreateBuffer(cl_context d_ctx, cl_mem_flags d_flags, size_t size,
89 void *host_ptr, cl_int *r_errcode) try {
90 const cl_mem_flags flags = validate_flags(NULL, d_flags, false);
91 auto &ctx = obj(d_ctx);
92
93 if (bool(host_ptr) != bool(flags & (CL_MEM_USE_HOST_PTR |
94 CL_MEM_COPY_HOST_PTR)))
95 throw error(CL_INVALID_HOST_PTR);
96
97 if (!size ||
98 size > fold(maximum(), cl_ulong(0),
99 map(std::mem_fn(&device::max_mem_alloc_size), ctx.devices())
100 ))
101 throw error(CL_INVALID_BUFFER_SIZE);
102
103 ret_error(r_errcode, CL_SUCCESS);
104 return new root_buffer(ctx, flags, size, host_ptr);
105
106 } catch (error &e) {
107 ret_error(r_errcode, e);
108 return NULL;
109 }
110
111 CLOVER_API cl_mem
clCreateSubBuffer(cl_mem d_mem,cl_mem_flags d_flags,cl_buffer_create_type op,const void * op_info,cl_int * r_errcode)112 clCreateSubBuffer(cl_mem d_mem, cl_mem_flags d_flags,
113 cl_buffer_create_type op,
114 const void *op_info, cl_int *r_errcode) try {
115 auto &parent = obj<root_buffer>(d_mem);
116 const cl_mem_flags flags = validate_flags(d_mem, d_flags, false);
117
118 if (op == CL_BUFFER_CREATE_TYPE_REGION) {
119 auto reg = reinterpret_cast<const cl_buffer_region *>(op_info);
120
121 if (!reg ||
122 reg->origin > parent.size() ||
123 reg->origin + reg->size > parent.size())
124 throw error(CL_INVALID_VALUE);
125
126 if (!reg->size)
127 throw error(CL_INVALID_BUFFER_SIZE);
128
129 ret_error(r_errcode, CL_SUCCESS);
130 return new sub_buffer(parent, flags, reg->origin, reg->size);
131
132 } else {
133 throw error(CL_INVALID_VALUE);
134 }
135
136 } catch (error &e) {
137 ret_error(r_errcode, e);
138 return NULL;
139 }
140
141 CLOVER_API cl_mem
clCreateImage(cl_context d_ctx,cl_mem_flags d_flags,const cl_image_format * format,const cl_image_desc * desc,void * host_ptr,cl_int * r_errcode)142 clCreateImage(cl_context d_ctx, cl_mem_flags d_flags,
143 const cl_image_format *format,
144 const cl_image_desc *desc,
145 void *host_ptr, cl_int *r_errcode) try {
146 auto &ctx = obj(d_ctx);
147
148 if (!any_of(std::mem_fn(&device::image_support), ctx.devices()))
149 throw error(CL_INVALID_OPERATION);
150
151 if (!format)
152 throw error(CL_INVALID_IMAGE_FORMAT_DESCRIPTOR);
153
154 if (!desc)
155 throw error(CL_INVALID_IMAGE_DESCRIPTOR);
156
157 if (desc->image_array_size == 0 &&
158 (desc->image_type == CL_MEM_OBJECT_IMAGE1D_ARRAY ||
159 desc->image_type == CL_MEM_OBJECT_IMAGE2D_ARRAY))
160 throw error(CL_INVALID_IMAGE_DESCRIPTOR);
161
162 if (!host_ptr &&
163 (desc->image_row_pitch || desc->image_slice_pitch))
164 throw error(CL_INVALID_IMAGE_DESCRIPTOR);
165
166 if (desc->num_mip_levels || desc->num_samples)
167 throw error(CL_INVALID_IMAGE_DESCRIPTOR);
168
169 if (bool(desc->buffer) != (desc->image_type == CL_MEM_OBJECT_IMAGE1D_BUFFER))
170 throw error(CL_INVALID_IMAGE_DESCRIPTOR);
171
172 if (bool(host_ptr) != bool(d_flags & (CL_MEM_USE_HOST_PTR |
173 CL_MEM_COPY_HOST_PTR)))
174 throw error(CL_INVALID_HOST_PTR);
175
176 const cl_mem_flags flags = validate_flags(desc->buffer, d_flags, false);
177
178 if (!supported_formats(ctx, desc->image_type).count(*format))
179 throw error(CL_IMAGE_FORMAT_NOT_SUPPORTED);
180
181 ret_error(r_errcode, CL_SUCCESS);
182
183 const size_t row_pitch = desc->image_row_pitch ? desc->image_row_pitch :
184 util_format_get_blocksize(translate_format(*format)) * desc->image_width;
185
186 switch (desc->image_type) {
187 case CL_MEM_OBJECT_IMAGE2D:
188 if (!desc->image_width || !desc->image_height)
189 throw error(CL_INVALID_IMAGE_SIZE);
190
191 if (all_of([=](const device &dev) {
192 const size_t max = 1 << dev.max_image_levels_2d();
193 return (desc->image_width > max ||
194 desc->image_height > max);
195 }, ctx.devices()))
196 throw error(CL_INVALID_IMAGE_SIZE);
197
198 return new image2d(ctx, flags, format,
199 desc->image_width, desc->image_height,
200 row_pitch, host_ptr);
201
202 case CL_MEM_OBJECT_IMAGE3D: {
203 if (!desc->image_width || !desc->image_height || !desc->image_depth)
204 throw error(CL_INVALID_IMAGE_SIZE);
205
206 if (all_of([=](const device &dev) {
207 const size_t max = 1 << dev.max_image_levels_3d();
208 return (desc->image_width > max ||
209 desc->image_height > max ||
210 desc->image_depth > max);
211 }, ctx.devices()))
212 throw error(CL_INVALID_IMAGE_SIZE);
213
214 const size_t slice_pitch = desc->image_slice_pitch ?
215 desc->image_slice_pitch : row_pitch * desc->image_height;
216
217 return new image3d(ctx, flags, format,
218 desc->image_width, desc->image_height,
219 desc->image_depth, row_pitch,
220 slice_pitch, host_ptr);
221 }
222
223 case CL_MEM_OBJECT_IMAGE1D:
224 case CL_MEM_OBJECT_IMAGE1D_ARRAY:
225 case CL_MEM_OBJECT_IMAGE1D_BUFFER:
226 case CL_MEM_OBJECT_IMAGE2D_ARRAY:
227 // XXX - Not implemented.
228 throw error(CL_IMAGE_FORMAT_NOT_SUPPORTED);
229
230 default:
231 throw error(CL_INVALID_IMAGE_DESCRIPTOR);
232 }
233
234 } catch (error &e) {
235 ret_error(r_errcode, e);
236 return NULL;
237 }
238
239 CLOVER_API cl_mem
clCreateImage2D(cl_context d_ctx,cl_mem_flags d_flags,const cl_image_format * format,size_t width,size_t height,size_t row_pitch,void * host_ptr,cl_int * r_errcode)240 clCreateImage2D(cl_context d_ctx, cl_mem_flags d_flags,
241 const cl_image_format *format,
242 size_t width, size_t height, size_t row_pitch,
243 void *host_ptr, cl_int *r_errcode) {
244 const cl_image_desc desc = { CL_MEM_OBJECT_IMAGE2D, width, height, 0, 0,
245 row_pitch, 0, 0, 0, NULL };
246
247 return clCreateImage(d_ctx, d_flags, format, &desc, host_ptr, r_errcode);
248 }
249
250 CLOVER_API cl_mem
clCreateImage3D(cl_context d_ctx,cl_mem_flags d_flags,const cl_image_format * format,size_t width,size_t height,size_t depth,size_t row_pitch,size_t slice_pitch,void * host_ptr,cl_int * r_errcode)251 clCreateImage3D(cl_context d_ctx, cl_mem_flags d_flags,
252 const cl_image_format *format,
253 size_t width, size_t height, size_t depth,
254 size_t row_pitch, size_t slice_pitch,
255 void *host_ptr, cl_int *r_errcode) {
256 const cl_image_desc desc = { CL_MEM_OBJECT_IMAGE3D, width, height, depth, 0,
257 row_pitch, slice_pitch, 0, 0, NULL };
258
259 return clCreateImage(d_ctx, d_flags, format, &desc, host_ptr, r_errcode);
260 }
261
262 CLOVER_API cl_int
clGetSupportedImageFormats(cl_context d_ctx,cl_mem_flags flags,cl_mem_object_type type,cl_uint count,cl_image_format * r_buf,cl_uint * r_count)263 clGetSupportedImageFormats(cl_context d_ctx, cl_mem_flags flags,
264 cl_mem_object_type type, cl_uint count,
265 cl_image_format *r_buf, cl_uint *r_count) try {
266 auto &ctx = obj(d_ctx);
267 auto formats = supported_formats(ctx, type);
268
269 validate_flags(NULL, flags, false);
270
271 if (r_buf && !count)
272 throw error(CL_INVALID_VALUE);
273
274 if (r_buf)
275 std::copy_n(formats.begin(),
276 std::min((cl_uint)formats.size(), count),
277 r_buf);
278
279 if (r_count)
280 *r_count = formats.size();
281
282 return CL_SUCCESS;
283
284 } catch (error &e) {
285 return e.get();
286 }
287
288 CLOVER_API cl_int
clGetMemObjectInfo(cl_mem d_mem,cl_mem_info param,size_t size,void * r_buf,size_t * r_size)289 clGetMemObjectInfo(cl_mem d_mem, cl_mem_info param,
290 size_t size, void *r_buf, size_t *r_size) try {
291 property_buffer buf { r_buf, size, r_size };
292 auto &mem = obj(d_mem);
293
294 switch (param) {
295 case CL_MEM_TYPE:
296 buf.as_scalar<cl_mem_object_type>() = mem.type();
297 break;
298
299 case CL_MEM_FLAGS:
300 buf.as_scalar<cl_mem_flags>() = mem.flags();
301 break;
302
303 case CL_MEM_SIZE:
304 buf.as_scalar<size_t>() = mem.size();
305 break;
306
307 case CL_MEM_HOST_PTR:
308 buf.as_scalar<void *>() = mem.host_ptr();
309 break;
310
311 case CL_MEM_MAP_COUNT:
312 buf.as_scalar<cl_uint>() = 0;
313 break;
314
315 case CL_MEM_REFERENCE_COUNT:
316 buf.as_scalar<cl_uint>() = mem.ref_count();
317 break;
318
319 case CL_MEM_CONTEXT:
320 buf.as_scalar<cl_context>() = desc(mem.context());
321 break;
322
323 case CL_MEM_ASSOCIATED_MEMOBJECT: {
324 sub_buffer *sub = dynamic_cast<sub_buffer *>(&mem);
325 buf.as_scalar<cl_mem>() = (sub ? desc(sub->parent()) : NULL);
326 break;
327 }
328 case CL_MEM_OFFSET: {
329 sub_buffer *sub = dynamic_cast<sub_buffer *>(&mem);
330 buf.as_scalar<size_t>() = (sub ? sub->offset() : 0);
331 break;
332 }
333 case CL_MEM_USES_SVM_POINTER:
334 case CL_MEM_USES_SVM_POINTER_ARM: {
335 // with system SVM all host ptrs are SVM pointers
336 // TODO: once we support devices with lower levels of SVM, we have to
337 // check the ptr in more detail
338 const bool system_svm = all_of(std::mem_fn(&device::has_system_svm),
339 mem.context().devices());
340 buf.as_scalar<cl_bool>() = mem.host_ptr() && system_svm;
341 break;
342 }
343 default:
344 throw error(CL_INVALID_VALUE);
345 }
346
347 return CL_SUCCESS;
348
349 } catch (error &e) {
350 return e.get();
351 }
352
353 CLOVER_API cl_int
clGetImageInfo(cl_mem d_mem,cl_image_info param,size_t size,void * r_buf,size_t * r_size)354 clGetImageInfo(cl_mem d_mem, cl_image_info param,
355 size_t size, void *r_buf, size_t *r_size) try {
356 property_buffer buf { r_buf, size, r_size };
357 auto &img = obj<image>(d_mem);
358
359 switch (param) {
360 case CL_IMAGE_FORMAT:
361 buf.as_scalar<cl_image_format>() = img.format();
362 break;
363
364 case CL_IMAGE_ELEMENT_SIZE:
365 buf.as_scalar<size_t>() = 0;
366 break;
367
368 case CL_IMAGE_ROW_PITCH:
369 buf.as_scalar<size_t>() = img.row_pitch();
370 break;
371
372 case CL_IMAGE_SLICE_PITCH:
373 buf.as_scalar<size_t>() = img.slice_pitch();
374 break;
375
376 case CL_IMAGE_WIDTH:
377 buf.as_scalar<size_t>() = img.width();
378 break;
379
380 case CL_IMAGE_HEIGHT:
381 buf.as_scalar<size_t>() = img.height();
382 break;
383
384 case CL_IMAGE_DEPTH:
385 buf.as_scalar<size_t>() = img.depth();
386 break;
387
388 default:
389 throw error(CL_INVALID_VALUE);
390 }
391
392 return CL_SUCCESS;
393
394 } catch (error &e) {
395 return e.get();
396 }
397
398 CLOVER_API cl_int
clRetainMemObject(cl_mem d_mem)399 clRetainMemObject(cl_mem d_mem) try {
400 obj(d_mem).retain();
401 return CL_SUCCESS;
402
403 } catch (error &e) {
404 return e.get();
405 }
406
407 CLOVER_API cl_int
clReleaseMemObject(cl_mem d_mem)408 clReleaseMemObject(cl_mem d_mem) try {
409 if (obj(d_mem).release())
410 delete pobj(d_mem);
411
412 return CL_SUCCESS;
413
414 } catch (error &e) {
415 return e.get();
416 }
417
418 CLOVER_API cl_int
clSetMemObjectDestructorCallback(cl_mem d_mem,void (CL_CALLBACK * pfn_notify)(cl_mem,void *),void * user_data)419 clSetMemObjectDestructorCallback(cl_mem d_mem,
420 void (CL_CALLBACK *pfn_notify)(cl_mem, void *),
421 void *user_data) try {
422 auto &mem = obj(d_mem);
423
424 if (!pfn_notify)
425 return CL_INVALID_VALUE;
426
427 mem.destroy_notify([=]{ pfn_notify(d_mem, user_data); });
428
429 return CL_SUCCESS;
430
431 } catch (error &e) {
432 return e.get();
433 }
434
435 CLOVER_API void *
clSVMAlloc(cl_context d_ctx,cl_svm_mem_flags flags,size_t size,unsigned int alignment)436 clSVMAlloc(cl_context d_ctx,
437 cl_svm_mem_flags flags,
438 size_t size,
439 unsigned int alignment) try {
440 auto &ctx = obj(d_ctx);
441 validate_flags(NULL, flags, true);
442
443 if (!size ||
444 size > fold(minimum(), cl_ulong(ULONG_MAX),
445 map(std::mem_fn(&device::max_mem_alloc_size), ctx.devices())))
446 return nullptr;
447
448 if (!util_is_power_of_two_or_zero(alignment))
449 return nullptr;
450
451 if (!alignment)
452 alignment = 0x80; // sizeof(long16)
453
454 bool can_emulate = all_of(std::mem_fn(&device::has_system_svm), ctx.devices());
455 if (can_emulate) {
456 // we can ignore all the flags as it's not required to honor them.
457 void *ptr = nullptr;
458 if (alignment < sizeof(void*))
459 alignment = sizeof(void*);
460 posix_memalign(&ptr, alignment, size);
461 return ptr;
462 }
463
464 CLOVER_NOT_SUPPORTED_UNTIL("2.0");
465 return nullptr;
466
467 } catch (error &e) {
468 return nullptr;
469 }
470
471 CLOVER_API void
clSVMFree(cl_context d_ctx,void * svm_pointer)472 clSVMFree(cl_context d_ctx,
473 void *svm_pointer) try {
474 auto &ctx = obj(d_ctx);
475 bool can_emulate = all_of(std::mem_fn(&device::has_system_svm), ctx.devices());
476
477 if (can_emulate)
478 return free(svm_pointer);
479
480 CLOVER_NOT_SUPPORTED_UNTIL("2.0");
481
482 } catch (error &e) {
483 }
484