clover/api/transfer.cpp

//
// Copyright 2012 Francisco Jerez
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
// OTHER DEALINGS IN THE SOFTWARE.
//

#include <cstring>

#include "util/bitscan.h"

#include "api/dispatch.hpp"
#include "api/util.hpp"
#include "core/event.hpp"
#include "core/memory.hpp"

using namespace clover;

namespace {
   typedef resource::vector vector_t;

   vector_t
   vector(const size_t *p) {
      if (!p)
         throw error(CL_INVALID_VALUE);
      return range(p, 3);
   }

   vector_t
   pitch(const vector_t &region, vector_t pitch) {
      for (auto x : zip(tail(pitch),
                        map(multiplies(), region, pitch))) {
         // The spec defines a value of zero as the natural pitch,
         // i.e. the unaligned size of the previous dimension.
         if (std::get<0>(x) == 0)
            std::get<0>(x) = std::get<1>(x);
      }

      return pitch;
   }

   ///
   /// Size of a region in bytes.
   ///
   size_t
   size(const vector_t &pitch, const vector_t &region) {
      if (any_of(is_zero(), region))
         return 0;
      else
         return dot(pitch, region - vector_t{ 0, 1, 1 });
   }

   ///
   /// Common argument checking shared by memory transfer commands.
   ///
   void
   validate_common(command_queue &q,
                   const ref_vector<event> &deps) {
      if (any_of([&](const event &ev) {
               return ev.context() != q.context();
            }, deps))
         throw error(CL_INVALID_CONTEXT);
   }

   ///
   /// Common error checking for a buffer object argument.
   ///
   void
   validate_object(command_queue &q, buffer &mem, const vector_t &origin,
                   const vector_t &pitch, const vector_t &region) {
      if (mem.context() != q.context())
         throw error(CL_INVALID_CONTEXT);

      // The region must fit within the specified pitch,
      if (any_of(greater(), map(multiplies(), pitch, region), tail(pitch)))
         throw error(CL_INVALID_VALUE);

      // ...and within the specified object.
      if (dot(pitch, origin) + size(pitch, region) > mem.size())
         throw error(CL_INVALID_VALUE);

      if (any_of(is_zero(), region))
         throw error(CL_INVALID_VALUE);
   }

   ///
   /// Common error checking for an image argument.
   ///
   void
   validate_object(command_queue &q, image &img,
                   const vector_t &orig, const vector_t &region) {
      size_t height = img.type() == CL_MEM_OBJECT_IMAGE1D_ARRAY ? img.array_size() : img.height();
      size_t depth = img.type() == CL_MEM_OBJECT_IMAGE2D_ARRAY ? img.array_size() : img.depth();
      vector_t size = { img.width(), height, depth };
      const auto &dev = q.device();

      if (!dev.image_support())
         throw error(CL_INVALID_OPERATION);

      if (img.context() != q.context())
         throw error(CL_INVALID_CONTEXT);

      if (any_of(greater(), orig + region, size))
         throw error(CL_INVALID_VALUE);

      if (any_of(is_zero(), region))
         throw error(CL_INVALID_VALUE);

      switch (img.type()) {
      case CL_MEM_OBJECT_IMAGE1D: {
         const size_t max = dev.max_image_size();
         if (img.width() > max)
            throw error(CL_INVALID_IMAGE_SIZE);
         break;
      }
      case CL_MEM_OBJECT_IMAGE1D_ARRAY: {
         const size_t max_size = dev.max_image_size();
         const size_t max_array = dev.max_image_array_number();
         if (img.width() > max_size || img.array_size() > max_array)
            throw error(CL_INVALID_IMAGE_SIZE);
         break;
      }
      case CL_MEM_OBJECT_IMAGE2D: {
         const size_t max = dev.max_image_size();
         if (img.width() > max || img.height() > max)
            throw error(CL_INVALID_IMAGE_SIZE);
         break;
      }
      case CL_MEM_OBJECT_IMAGE2D_ARRAY: {
         const size_t max_size = dev.max_image_size();
         const size_t max_array = dev.max_image_array_number();
         if (img.width() > max_size || img.height() > max_size || img.array_size() > max_array)
            throw error(CL_INVALID_IMAGE_SIZE);
         break;
      }
      case CL_MEM_OBJECT_IMAGE3D: {
         const size_t max = dev.max_image_size_3d();
         if (img.width() > max || img.height() > max || img.depth() > max)
            throw error(CL_INVALID_IMAGE_SIZE);
         break;
      }
      // XXX: Implement missing checks once Clover supports more image types.
      default:
         throw error(CL_INVALID_IMAGE_SIZE);
      }
   }

   ///
   /// Common error checking for a host pointer argument.
   ///
   void
   validate_object(command_queue &q, const void *ptr, const vector_t &orig,
                   const vector_t &pitch, const vector_t &region) {
      if (!ptr)
         throw error(CL_INVALID_VALUE);

      // The region must fit within the specified pitch.
      if (any_of(greater(), map(multiplies(), pitch, region), tail(pitch)))
         throw error(CL_INVALID_VALUE);
   }

   ///
   /// Common argument checking for a copy between two buffer objects.
   ///
   void
   validate_copy(command_queue &q, buffer &dst_mem,
                 const vector_t &dst_orig, const vector_t &dst_pitch,
                 buffer &src_mem,
                 const vector_t &src_orig, const vector_t &src_pitch,
                 const vector_t &region) {
      if (dst_mem == src_mem) {
         auto dst_offset = dot(dst_pitch, dst_orig);
         auto src_offset = dot(src_pitch, src_orig);

         if (interval_overlaps()(
                dst_offset, dst_offset + size(dst_pitch, region),
                src_offset, src_offset + size(src_pitch, region)))
            throw error(CL_MEM_COPY_OVERLAP);
      }
   }

   ///
   /// Common argument checking for a copy between two image objects.
   ///
   void
   validate_copy(command_queue &q,
                 image &dst_img, const vector_t &dst_orig,
                 image &src_img, const vector_t &src_orig,
                 const vector_t &region) {
      if (dst_img.format() != src_img.format())
         throw error(CL_IMAGE_FORMAT_MISMATCH);

      if (dst_img == src_img) {
         if (all_of(interval_overlaps(),
                    dst_orig, dst_orig + region,
                    src_orig, src_orig + region))
            throw error(CL_MEM_COPY_OVERLAP);
      }
   }

   ///
   /// Checks that the host access flags of the memory object are
   /// within the allowed set \a flags.
   ///
   void
   validate_object_access(const memory_obj &mem, const cl_mem_flags flags) {
      if (mem.flags() & ~flags &
          (CL_MEM_HOST_READ_ONLY | CL_MEM_HOST_WRITE_ONLY |
           CL_MEM_HOST_NO_ACCESS))
         throw error(CL_INVALID_OPERATION);
   }

   ///
   /// Checks that the mapping flags are correct.
   ///
   void
   validate_map_flags(const memory_obj &mem, const cl_map_flags flags) {
      if ((flags & (CL_MAP_WRITE | CL_MAP_READ)) &&
          (flags & CL_MAP_WRITE_INVALIDATE_REGION))
         throw error(CL_INVALID_VALUE);

      if (flags & CL_MAP_READ)
         validate_object_access(mem, CL_MEM_HOST_READ_ONLY);

      if (flags & (CL_MAP_WRITE | CL_MAP_WRITE_INVALIDATE_REGION))
         validate_object_access(mem, CL_MEM_HOST_WRITE_ONLY);
   }

   ///
   /// Checks that the memory migration flags are correct.
   ///
   void
   validate_mem_migration_flags(const cl_mem_migration_flags flags) {
      const cl_mem_migration_flags valid =
         CL_MIGRATE_MEM_OBJECT_HOST |
         CL_MIGRATE_MEM_OBJECT_CONTENT_UNDEFINED;

      if (flags & ~valid)
         throw error(CL_INVALID_VALUE);
   }

   ///
   /// Class that encapsulates the task of mapping an object of type
   /// \a T.  The return value of get() should be implicitly
   /// convertible to \a void *.
   ///
   template<typename T>
   struct _map;

   template<>
   struct _map<image*> {
      _map(command_queue &q, image *img, cl_map_flags flags,
           vector_t offset, vector_t pitch, vector_t region) :
         map(q, img->resource_in(q), flags, true, offset, region),
         pitch(map.pitch())
      { }

      template<typename T>
      operator T *() const {
         return static_cast<T *>(map);
      }

      mapping map;
      vector_t pitch;
   };

   template<>
   struct _map<buffer*> {
      _map(command_queue &q, buffer *mem, cl_map_flags flags,
           vector_t offset, vector_t pitch, vector_t region) :
         map(q, mem->resource_in(q), flags, true,
             {{ dot(pitch, offset) }}, {{ size(pitch, region) }}),
         pitch(pitch)
      { }

      template<typename T>
      operator T *() const {
         return static_cast<T *>(map);
      }

      mapping map;
      vector_t pitch;
   };

   template<typename P>
   struct _map<P *> {
      _map(command_queue &q, P *ptr, cl_map_flags flags,
           vector_t offset, vector_t pitch, vector_t region) :
         ptr((P *)((char *)ptr + dot(pitch, offset))), pitch(pitch)
      { }

      template<typename T>
      operator T *() const {
         return static_cast<T *>(ptr);
      }

      P *ptr;
      vector_t pitch;
   };

   ///
   /// Software copy from \a src_obj to \a dst_obj.  They can be
   /// either pointers or memory objects.
   ///
   template<typename T, typename S>
   std::function<void (event &)>
   soft_copy_op(command_queue &q,
                T dst_obj, const vector_t &dst_orig, const vector_t &dst_pitch,
                S src_obj, const vector_t &src_orig, const vector_t &src_pitch,
                const vector_t &region) {
      return [=, &q](event &) {
         _map<T> dst = { q, dst_obj, CL_MAP_WRITE,
                         dst_orig, dst_pitch, region };
         _map<S> src = { q, src_obj, CL_MAP_READ,
                         src_orig, src_pitch, region };
         assert(src.pitch[0] == dst.pitch[0]);
         vector_t v = {};

         for (v[2] = 0; v[2] < region[2]; ++v[2]) {
            for (v[1] = 0; v[1] < region[1]; ++v[1]) {
               std::memcpy(
                  static_cast<char *>(dst) + dot(dst.pitch, v),
                  static_cast<const char *>(src) + dot(src.pitch, v),
                  src.pitch[0] * region[0]);
            }
         }
      };
   }

   ///
   /// Hardware copy from \a src_obj to \a dst_obj.
   ///
   template<typename T, typename S>
   std::function<void (event &)>
   hard_copy_op(command_queue &q, T dst_obj, const vector_t &dst_orig,
                S src_obj, const vector_t &src_orig, const vector_t &region) {
      return [=, &q](event &) {
         dst_obj->resource_in(q).copy(q, dst_orig, region,
                                      src_obj->resource_in(q), src_orig);
      };
   }
}

CLOVER_API cl_int
clEnqueueReadBuffer(cl_command_queue d_q, cl_mem d_mem, cl_bool blocking,
                    size_t offset, size_t size, void *ptr,
                    cl_uint num_deps, const cl_event *d_deps,
                    cl_event *rd_ev) try {
   auto &q = obj(d_q);
   auto &mem = obj<buffer>(d_mem);
   auto deps = objs<wait_list_tag>(d_deps, num_deps);
   vector_t region = { size, 1, 1 };
   vector_t obj_origin = { offset };
   auto obj_pitch = pitch(region, {{ 1 }});

   validate_common(q, deps);
   validate_object(q, ptr, {}, obj_pitch, region);
   validate_object(q, mem, obj_origin, obj_pitch, region);
   validate_object_access(mem, CL_MEM_HOST_READ_ONLY);

   auto hev = create<hard_event>(
      q, CL_COMMAND_READ_BUFFER, deps,
      soft_copy_op(q, ptr, {}, obj_pitch,
                   &mem, obj_origin, obj_pitch,
                   region));

   if (blocking)
       hev().wait_signalled();

   ret_object(rd_ev, hev);
   return CL_SUCCESS;

} catch (error &e) {
   return e.get();
}

CLOVER_API cl_int
clEnqueueWriteBuffer(cl_command_queue d_q, cl_mem d_mem, cl_bool blocking,
                     size_t offset, size_t size, const void *ptr,
                     cl_uint num_deps, const cl_event *d_deps,
                     cl_event *rd_ev) try {
   auto &q = obj(d_q);
   auto &mem = obj<buffer>(d_mem);
   auto deps = objs<wait_list_tag>(d_deps, num_deps);
   vector_t region = { size, 1, 1 };
   vector_t obj_origin = { offset };
   auto obj_pitch = pitch(region, {{ 1 }});

   validate_common(q, deps);
   validate_object(q, mem, obj_origin, obj_pitch, region);
   validate_object(q, ptr, {}, obj_pitch, region);
   validate_object_access(mem, CL_MEM_HOST_WRITE_ONLY);

   auto hev = create<hard_event>(
      q, CL_COMMAND_WRITE_BUFFER, deps,
      soft_copy_op(q, &mem, obj_origin, obj_pitch,
                   ptr, {}, obj_pitch,
                   region));

   if (blocking)
       hev().wait_signalled();

   ret_object(rd_ev, hev);
   return CL_SUCCESS;

} catch (error &e) {
   return e.get();
}

CLOVER_API cl_int
clEnqueueReadBufferRect(cl_command_queue d_q, cl_mem d_mem, cl_bool blocking,
                        const size_t *p_obj_origin,
                        const size_t *p_host_origin,
                        const size_t *p_region,
                        size_t obj_row_pitch, size_t obj_slice_pitch,
                        size_t host_row_pitch, size_t host_slice_pitch,
                        void *ptr,
                        cl_uint num_deps, const cl_event *d_deps,
                        cl_event *rd_ev) try {
   auto &q = obj(d_q);
   auto &mem = obj<buffer>(d_mem);
   auto deps = objs<wait_list_tag>(d_deps, num_deps);
   auto region = vector(p_region);
   auto obj_origin = vector(p_obj_origin);
   auto obj_pitch = pitch(region, {{ 1, obj_row_pitch, obj_slice_pitch }});
   auto host_origin = vector(p_host_origin);
   auto host_pitch = pitch(region, {{ 1, host_row_pitch, host_slice_pitch }});

   validate_common(q, deps);
   validate_object(q, ptr, host_origin, host_pitch, region);
   validate_object(q, mem, obj_origin, obj_pitch, region);
   validate_object_access(mem, CL_MEM_HOST_READ_ONLY);

   auto hev = create<hard_event>(
      q, CL_COMMAND_READ_BUFFER_RECT, deps,
      soft_copy_op(q, ptr, host_origin, host_pitch,
                   &mem, obj_origin, obj_pitch,
                   region));

   if (blocking)
       hev().wait_signalled();

   ret_object(rd_ev, hev);
   return CL_SUCCESS;

} catch (error &e) {
   return e.get();
}

CLOVER_API cl_int
clEnqueueWriteBufferRect(cl_command_queue d_q, cl_mem d_mem, cl_bool blocking,
                         const size_t *p_obj_origin,
                         const size_t *p_host_origin,
                         const size_t *p_region,
                         size_t obj_row_pitch, size_t obj_slice_pitch,
                         size_t host_row_pitch, size_t host_slice_pitch,
                         const void *ptr,
                         cl_uint num_deps, const cl_event *d_deps,
                         cl_event *rd_ev) try {
   auto &q = obj(d_q);
   auto &mem = obj<buffer>(d_mem);
   auto deps = objs<wait_list_tag>(d_deps, num_deps);
   auto region = vector(p_region);
   auto obj_origin = vector(p_obj_origin);
   auto obj_pitch = pitch(region, {{ 1, obj_row_pitch, obj_slice_pitch }});
   auto host_origin = vector(p_host_origin);
   auto host_pitch = pitch(region, {{ 1, host_row_pitch, host_slice_pitch }});

   validate_common(q, deps);
   validate_object(q, mem, obj_origin, obj_pitch, region);
   validate_object(q, ptr, host_origin, host_pitch, region);
   validate_object_access(mem, CL_MEM_HOST_WRITE_ONLY);

   auto hev = create<hard_event>(
      q, CL_COMMAND_WRITE_BUFFER_RECT, deps,
      soft_copy_op(q, &mem, obj_origin, obj_pitch,
                   ptr, host_origin, host_pitch,
                   region));

   if (blocking)
       hev().wait_signalled();

   ret_object(rd_ev, hev);
   return CL_SUCCESS;

} catch (error &e) {
   return e.get();
}

CLOVER_API cl_int
clEnqueueFillBuffer(cl_command_queue d_queue, cl_mem d_mem,
                    const void *pattern, size_t pattern_size,
                    size_t offset, size_t size,
                    cl_uint num_deps, const cl_event *d_deps,
                    cl_event *rd_ev) try {
   auto &q = obj(d_queue);
   auto &mem = obj<buffer>(d_mem);
   auto deps = objs<wait_list_tag>(d_deps, num_deps);
   vector_t region = { size, 1, 1 };
   vector_t origin = { offset };
   auto dst_pitch = pitch(region, {{ 1 }});

   validate_common(q, deps);
   validate_object(q, mem, origin, dst_pitch, region);

   if (!pattern)
      return CL_INVALID_VALUE;

   if (!util_is_power_of_two_nonzero(pattern_size) ||
      pattern_size > 128 || size % pattern_size
      || offset % pattern_size) {
      return CL_INVALID_VALUE;
   }

   auto sub = dynamic_cast<sub_buffer *>(&mem);
   if (sub && sub->offset() % q.device().mem_base_addr_align()) {
      return CL_MISALIGNED_SUB_BUFFER_OFFSET;
   }

   std::string data = std::string((char *)pattern, pattern_size);
   auto hev = create<hard_event>(
      q, CL_COMMAND_FILL_BUFFER, deps,
      [=, &q, &mem](event &) {
         mem.resource_in(q).clear(q, origin, region, data);
      });

   ret_object(rd_ev, hev);
   return CL_SUCCESS;

} catch (error &e) {
   return e.get();
}

CLOVER_API cl_int
clEnqueueCopyBuffer(cl_command_queue d_q, cl_mem d_src_mem, cl_mem d_dst_mem,
                    size_t src_offset, size_t dst_offset, size_t size,
                    cl_uint num_deps, const cl_event *d_deps,
                    cl_event *rd_ev) try {
   auto &q = obj(d_q);
   auto &src_mem = obj<buffer>(d_src_mem);
   auto &dst_mem = obj<buffer>(d_dst_mem);
   auto deps = objs<wait_list_tag>(d_deps, num_deps);
   vector_t region = { size, 1, 1 };
   vector_t dst_origin = { dst_offset };
   auto dst_pitch = pitch(region, {{ 1 }});
   vector_t src_origin = { src_offset };
   auto src_pitch = pitch(region, {{ 1 }});

   validate_common(q, deps);
   validate_object(q, dst_mem, dst_origin, dst_pitch, region);
   validate_object(q, src_mem, src_origin, src_pitch, region);
   validate_copy(q, dst_mem, dst_origin, dst_pitch,
                 src_mem, src_origin, src_pitch, region);

   auto hev = create<hard_event>(
      q, CL_COMMAND_COPY_BUFFER, deps,
      hard_copy_op(q, &dst_mem, dst_origin,
                   &src_mem, src_origin, region));

   ret_object(rd_ev, hev);
   return CL_SUCCESS;

} catch (error &e) {
   return e.get();
}

CLOVER_API cl_int
clEnqueueCopyBufferRect(cl_command_queue d_q, cl_mem d_src_mem,
                        cl_mem d_dst_mem,
                        const size_t *p_src_origin, const size_t *p_dst_origin,
                        const size_t *p_region,
                        size_t src_row_pitch, size_t src_slice_pitch,
                        size_t dst_row_pitch, size_t dst_slice_pitch,
                        cl_uint num_deps, const cl_event *d_deps,
                        cl_event *rd_ev) try {
   auto &q = obj(d_q);
   auto &src_mem = obj<buffer>(d_src_mem);
   auto &dst_mem = obj<buffer>(d_dst_mem);
   auto deps = objs<wait_list_tag>(d_deps, num_deps);
   auto region = vector(p_region);
   auto dst_origin = vector(p_dst_origin);
   auto dst_pitch = pitch(region, {{ 1, dst_row_pitch, dst_slice_pitch }});
   auto src_origin = vector(p_src_origin);
   auto src_pitch = pitch(region, {{ 1, src_row_pitch, src_slice_pitch }});

   validate_common(q, deps);
   validate_object(q, dst_mem, dst_origin, dst_pitch, region);
   validate_object(q, src_mem, src_origin, src_pitch, region);
   validate_copy(q, dst_mem, dst_origin, dst_pitch,
                 src_mem, src_origin, src_pitch, region);

   auto hev = create<hard_event>(
      q, CL_COMMAND_COPY_BUFFER_RECT, deps,
      soft_copy_op(q, &dst_mem, dst_origin, dst_pitch,
                   &src_mem, src_origin, src_pitch,
                   region));

   ret_object(rd_ev, hev);
   return CL_SUCCESS;

} catch (error &e) {
   return e.get();
}

CLOVER_API cl_int
clEnqueueReadImage(cl_command_queue d_q, cl_mem d_mem, cl_bool blocking,
                   const size_t *p_origin, const size_t *p_region,
                   size_t row_pitch, size_t slice_pitch, void *ptr,
                   cl_uint num_deps, const cl_event *d_deps,
                   cl_event *rd_ev) try {
   auto &q = obj(d_q);
   auto &img = obj<image>(d_mem);
   auto deps = objs<wait_list_tag>(d_deps, num_deps);
   auto region = vector(p_region);
   auto dst_pitch = pitch(region, {{ img.pixel_size(),
                                     row_pitch, slice_pitch }});
   auto src_origin = vector(p_origin);
   auto src_pitch = pitch(region, {{ img.pixel_size(),
                                     img.row_pitch(), img.slice_pitch() }});

   validate_common(q, deps);
   validate_object(q, ptr, {}, dst_pitch, region);
   validate_object(q, img, src_origin, region);
   validate_object_access(img, CL_MEM_HOST_READ_ONLY);

   auto hev = create<hard_event>(
      q, CL_COMMAND_READ_IMAGE, deps,
      soft_copy_op(q, ptr, {}, dst_pitch,
                   &img, src_origin, src_pitch,
                   region));

   if (blocking)
       hev().wait_signalled();

   ret_object(rd_ev, hev);
   return CL_SUCCESS;

} catch (error &e) {
   return e.get();
}

CLOVER_API cl_int
clEnqueueWriteImage(cl_command_queue d_q, cl_mem d_mem, cl_bool blocking,
                    const size_t *p_origin, const size_t *p_region,
                    size_t row_pitch, size_t slice_pitch, const void *ptr,
                    cl_uint num_deps, const cl_event *d_deps,
                    cl_event *rd_ev) try {
   auto &q = obj(d_q);
   auto &img = obj<image>(d_mem);
   auto deps = objs<wait_list_tag>(d_deps, num_deps);
   auto region = vector(p_region);
   auto dst_origin = vector(p_origin);
   auto dst_pitch = pitch(region, {{ img.pixel_size(),
                                     img.row_pitch(), img.slice_pitch() }});
   auto src_pitch = pitch(region, {{ img.pixel_size(),
                                     row_pitch, slice_pitch }});

   validate_common(q, deps);
   validate_object(q, img, dst_origin, region);
   validate_object(q, ptr, {}, src_pitch, region);
   validate_object_access(img, CL_MEM_HOST_WRITE_ONLY);

   auto hev = create<hard_event>(
      q, CL_COMMAND_WRITE_IMAGE, deps,
      soft_copy_op(q, &img, dst_origin, dst_pitch,
                   ptr, {}, src_pitch,
                   region));

   if (blocking)
       hev().wait_signalled();

   ret_object(rd_ev, hev);
   return CL_SUCCESS;

} catch (error &e) {
   return e.get();
}

CLOVER_API cl_int
clEnqueueFillImage(cl_command_queue d_queue, cl_mem d_mem,
                   const void *fill_color,
                   const size_t *p_origin, const size_t *p_region,
                   cl_uint num_deps, const cl_event *d_deps,
                   cl_event *rd_ev) try {
   auto &q = obj(d_queue);
   auto &img = obj<image>(d_mem);
   auto deps = objs<wait_list_tag>(d_deps, num_deps);
   auto origin = vector(p_origin);
   auto region = vector(p_region);

   validate_common(q, deps);
   validate_object(q, img, origin, region);

   if (!fill_color)
      return CL_INVALID_VALUE;

   std::string data = std::string((char *)fill_color, sizeof(cl_uint4));
   auto hev = create<hard_event>(
      q, CL_COMMAND_FILL_IMAGE, deps,
      [=, &q, &img](event &) {
         img.resource_in(q).clear(q, origin, region, data);
      });

   ret_object(rd_ev, hev);
   return CL_SUCCESS;

} catch (error &e) {
   return e.get();
}

CLOVER_API cl_int
clEnqueueCopyImage(cl_command_queue d_q, cl_mem d_src_mem, cl_mem d_dst_mem,
                   const size_t *p_src_origin, const size_t *p_dst_origin,
                   const size_t *p_region,
                   cl_uint num_deps, const cl_event *d_deps,
                   cl_event *rd_ev) try {
   auto &q = obj(d_q);
   auto &src_img = obj<image>(d_src_mem);
   auto &dst_img = obj<image>(d_dst_mem);
   auto deps = objs<wait_list_tag>(d_deps, num_deps);
   auto region = vector(p_region);
   auto dst_origin = vector(p_dst_origin);
   auto src_origin = vector(p_src_origin);

   validate_common(q, deps);
   validate_object(q, dst_img, dst_origin, region);
   validate_object(q, src_img, src_origin, region);
   validate_copy(q, dst_img, dst_origin, src_img, src_origin, region);

   auto hev = create<hard_event>(
      q, CL_COMMAND_COPY_IMAGE, deps,
      hard_copy_op(q, &dst_img, dst_origin,
                   &src_img, src_origin,
                   region));

   ret_object(rd_ev, hev);
   return CL_SUCCESS;

} catch (error &e) {
   return e.get();
}

CLOVER_API cl_int
clEnqueueCopyImageToBuffer(cl_command_queue d_q,
                           cl_mem d_src_mem, cl_mem d_dst_mem,
                           const size_t *p_src_origin, const size_t *p_region,
                           size_t dst_offset,
                           cl_uint num_deps, const cl_event *d_deps,
                           cl_event *rd_ev) try {
   auto &q = obj(d_q);
   auto &src_img = obj<image>(d_src_mem);
   auto &dst_mem = obj<buffer>(d_dst_mem);
   auto deps = objs<wait_list_tag>(d_deps, num_deps);
   auto region = vector(p_region);
   vector_t dst_origin = { dst_offset };
   auto dst_pitch = pitch(region, {{ src_img.pixel_size() }});
   auto src_origin = vector(p_src_origin);
   auto src_pitch = pitch(region, {{ src_img.pixel_size(),
                                     src_img.row_pitch(),
                                     src_img.slice_pitch() }});

   validate_common(q, deps);
   validate_object(q, dst_mem, dst_origin, dst_pitch, region);
   validate_object(q, src_img, src_origin, region);

   auto hev = create<hard_event>(
      q, CL_COMMAND_COPY_IMAGE_TO_BUFFER, deps,
      soft_copy_op(q, &dst_mem, dst_origin, dst_pitch,
                   &src_img, src_origin, src_pitch,
                   region));

   ret_object(rd_ev, hev);
   return CL_SUCCESS;

} catch (error &e) {
   return e.get();
}

CLOVER_API cl_int
clEnqueueCopyBufferToImage(cl_command_queue d_q,
                           cl_mem d_src_mem, cl_mem d_dst_mem,
                           size_t src_offset,
                           const size_t *p_dst_origin, const size_t *p_region,
                           cl_uint num_deps, const cl_event *d_deps,
                           cl_event *rd_ev) try {
   auto &q = obj(d_q);
   auto &src_mem = obj<buffer>(d_src_mem);
   auto &dst_img = obj<image>(d_dst_mem);
   auto deps = objs<wait_list_tag>(d_deps, num_deps);
   auto region = vector(p_region);
   auto dst_origin = vector(p_dst_origin);
   auto dst_pitch = pitch(region, {{ dst_img.pixel_size(),
                                     dst_img.row_pitch(),
                                     dst_img.slice_pitch() }});
   vector_t src_origin = { src_offset };
   auto src_pitch = pitch(region, {{ dst_img.pixel_size() }});

   validate_common(q, deps);
   validate_object(q, dst_img, dst_origin, region);
   validate_object(q, src_mem, src_origin, src_pitch, region);

   auto hev = create<hard_event>(
      q, CL_COMMAND_COPY_BUFFER_TO_IMAGE, deps,
      soft_copy_op(q, &dst_img, dst_origin, dst_pitch,
                   &src_mem, src_origin, src_pitch,
                   region));

   ret_object(rd_ev, hev);
   return CL_SUCCESS;

} catch (error &e) {
   return e.get();
}

CLOVER_API void *
clEnqueueMapBuffer(cl_command_queue d_q, cl_mem d_mem, cl_bool blocking,
                   cl_map_flags flags, size_t offset, size_t size,
                   cl_uint num_deps, const cl_event *d_deps,
                   cl_event *rd_ev, cl_int *r_errcode) try {
   auto &q = obj(d_q);
   auto &mem = obj<buffer>(d_mem);
   auto deps = objs<wait_list_tag>(d_deps, num_deps);
   vector_t region = { size, 1, 1 };
   vector_t obj_origin = { offset };
   auto obj_pitch = pitch(region, {{ 1 }});

   validate_common(q, deps);
   validate_object(q, mem, obj_origin, obj_pitch, region);
   validate_map_flags(mem, flags);

   auto *map = mem.resource_in(q).add_map(q, flags, blocking, obj_origin, region);

   auto hev = create<hard_event>(q, CL_COMMAND_MAP_BUFFER, deps);
   if (blocking)
       hev().wait_signalled();

   ret_object(rd_ev, hev);
   ret_error(r_errcode, CL_SUCCESS);
   return *map;

} catch (error &e) {
   ret_error(r_errcode, e);
   return NULL;
}

CLOVER_API void *
clEnqueueMapImage(cl_command_queue d_q, cl_mem d_mem, cl_bool blocking,
                  cl_map_flags flags,
                  const size_t *p_origin, const size_t *p_region,
                  size_t *row_pitch, size_t *slice_pitch,
                  cl_uint num_deps, const cl_event *d_deps,
                  cl_event *rd_ev, cl_int *r_errcode) try {
   auto &q = obj(d_q);
   auto &img = obj<image>(d_mem);
   auto deps = objs<wait_list_tag>(d_deps, num_deps);
   auto region = vector(p_region);
   auto origin = vector(p_origin);

   validate_common(q, deps);
   validate_object(q, img, origin, region);
   validate_map_flags(img, flags);

   if (!row_pitch)
      throw error(CL_INVALID_VALUE);

   if ((img.slice_pitch() || img.array_size()) && !slice_pitch)
      throw error(CL_INVALID_VALUE);

   auto *map = img.resource_in(q).add_map(q, flags, blocking, origin, region);
   *row_pitch = map->pitch()[1];
   if (slice_pitch)
      *slice_pitch = map->pitch()[2];

   auto hev = create<hard_event>(q, CL_COMMAND_MAP_IMAGE, deps);
   if (blocking)
       hev().wait_signalled();

   ret_object(rd_ev, hev);
   ret_error(r_errcode, CL_SUCCESS);
   return *map;

} catch (error &e) {
   ret_error(r_errcode, e);
   return NULL;
}

CLOVER_API cl_int
clEnqueueUnmapMemObject(cl_command_queue d_q, cl_mem d_mem, void *ptr,
                        cl_uint num_deps, const cl_event *d_deps,
                        cl_event *rd_ev) try {
   auto &q = obj(d_q);
   auto &mem = obj(d_mem);
   auto deps = objs<wait_list_tag>(d_deps, num_deps);

   validate_common(q, deps);

   auto hev = create<hard_event>(
      q, CL_COMMAND_UNMAP_MEM_OBJECT, deps,
      [=, &q, &mem](event &) {
         mem.resource_in(q).del_map(ptr);
      });

   ret_object(rd_ev, hev);
   return CL_SUCCESS;

} catch (error &e) {
   return e.get();
}

CLOVER_API cl_int
clEnqueueMigrateMemObjects(cl_command_queue d_q,
                           cl_uint num_mems,
                           const cl_mem *d_mems,
                           cl_mem_migration_flags flags,
                           cl_uint num_deps,
                           const cl_event *d_deps,
                           cl_event *rd_ev) try {
   auto &q = obj(d_q);
   auto mems = objs<memory_obj>(d_mems, num_mems);
   auto deps = objs<wait_list_tag>(d_deps, num_deps);

   validate_common(q, deps);
   validate_mem_migration_flags(flags);

   if (any_of([&](const memory_obj &m) {
         return m.context() != q.context();
         }, mems))
      throw error(CL_INVALID_CONTEXT);

   auto hev = create<hard_event>(
      q, CL_COMMAND_MIGRATE_MEM_OBJECTS, deps,
      [=, &q](event &) {
         for (auto &mem: mems) {
            if (flags & CL_MIGRATE_MEM_OBJECT_HOST) {
               if ((flags & CL_MIGRATE_MEM_OBJECT_CONTENT_UNDEFINED))
                  mem.resource_out(q);

               // For flags == CL_MIGRATE_MEM_OBJECT_HOST only to be
               // efficient we would need cl*ReadBuffer* to implement
               // reading from host memory.

            } else {
               if (flags & CL_MIGRATE_MEM_OBJECT_CONTENT_UNDEFINED)
                  mem.resource_undef(q);
               else
                  mem.resource_in(q);
            }
         }
      });

   ret_object(rd_ev, hev);
   return CL_SUCCESS;;

} catch (error &e) {
   return e.get();
}

cl_int
clover::EnqueueSVMFree(cl_command_queue d_q,
                       cl_uint num_svm_pointers,
                       void *svm_pointers[],
                       void (CL_CALLBACK *pfn_free_func) (
                           cl_command_queue queue, cl_uint num_svm_pointers,
                           void *svm_pointers[], void *user_data),
                       void *user_data,
                       cl_uint num_events_in_wait_list,
                       const cl_event *event_wait_list,
                       cl_event *event,
                       cl_int cmd) try {

   if (bool(num_svm_pointers) != bool(svm_pointers))
      return CL_INVALID_VALUE;

   auto &q = obj(d_q);

   if (!q.device().svm_support())
      return CL_INVALID_OPERATION;

   bool can_emulate = q.device().has_system_svm();
   auto deps = objs<wait_list_tag>(event_wait_list, num_events_in_wait_list);

   validate_common(q, deps);

   std::vector<void *> svm_pointers_cpy(svm_pointers,
                                        svm_pointers + num_svm_pointers);
   if (!pfn_free_func) {
      if (!can_emulate) {
         CLOVER_NOT_SUPPORTED_UNTIL("2.0");
         return CL_INVALID_VALUE;
      }
      pfn_free_func = [](cl_command_queue d_q, cl_uint num_svm_pointers,
                         void *svm_pointers[], void *) {
         clover::context &ctx = obj(d_q).context();
         for (void *p : range(svm_pointers, num_svm_pointers)) {
            ctx.remove_svm_allocation(p);
            free(p);
         }
      };
   }

   auto hev = create<hard_event>(q, cmd, deps,
      [=](clover::event &) mutable {
         pfn_free_func(d_q, num_svm_pointers, svm_pointers_cpy.data(),
                       user_data);
      });

   ret_object(event, hev);
   return CL_SUCCESS;

} catch (error &e) {
   return e.get();
}

CLOVER_API cl_int
clEnqueueSVMFree(cl_command_queue d_q,
                 cl_uint num_svm_pointers,
                 void *svm_pointers[],
                 void (CL_CALLBACK *pfn_free_func) (
                    cl_command_queue queue, cl_uint num_svm_pointers,
                    void *svm_pointers[], void *user_data),
                 void *user_data,
                 cl_uint num_events_in_wait_list,
                 const cl_event *event_wait_list,
                 cl_event *event) {

   return EnqueueSVMFree(d_q, num_svm_pointers, svm_pointers,
                         pfn_free_func, user_data, num_events_in_wait_list,
                         event_wait_list, event, CL_COMMAND_SVM_FREE);
}

cl_int
clover::EnqueueSVMMemcpy(cl_command_queue d_q,
                         cl_bool blocking_copy,
                         void *dst_ptr,
                         const void *src_ptr,
                         size_t size,
                         cl_uint num_events_in_wait_list,
                         const cl_event *event_wait_list,
                         cl_event *event,
                         cl_int cmd) try {
   auto &q = obj(d_q);

   if (!q.device().svm_support())
      return CL_INVALID_OPERATION;

   if (dst_ptr == nullptr || src_ptr == nullptr)
      return CL_INVALID_VALUE;

   if (static_cast<size_t>(abs(reinterpret_cast<ptrdiff_t>(dst_ptr) -
                               reinterpret_cast<ptrdiff_t>(src_ptr))) < size)
      return CL_MEM_COPY_OVERLAP;


   bool can_emulate = q.device().has_system_svm();
   auto deps = objs<wait_list_tag>(event_wait_list, num_events_in_wait_list);

   validate_common(q, deps);

   if (can_emulate) {
      auto hev = create<hard_event>(q, cmd, deps,
         [=](clover::event &) {
            memcpy(dst_ptr, src_ptr, size);
         });

      if (blocking_copy)
         hev().wait();
      ret_object(event, hev);
      return CL_SUCCESS;
   }

   CLOVER_NOT_SUPPORTED_UNTIL("2.0");
   return CL_INVALID_VALUE;

} catch (error &e) {
   return e.get();
}

CLOVER_API cl_int
clEnqueueSVMMemcpy(cl_command_queue d_q,
                   cl_bool blocking_copy,
                   void *dst_ptr,
                   const void *src_ptr,
                   size_t size,
                   cl_uint num_events_in_wait_list,
                   const cl_event *event_wait_list,
                   cl_event *event) {

   return EnqueueSVMMemcpy(d_q, blocking_copy, dst_ptr, src_ptr,
                           size, num_events_in_wait_list, event_wait_list,
                           event, CL_COMMAND_SVM_MEMCPY);
}

cl_int
clover::EnqueueSVMMemFill(cl_command_queue d_q,
                          void *svm_ptr,
                          const void *pattern,
                          size_t pattern_size,
                          size_t size,
                          cl_uint num_events_in_wait_list,
                          const cl_event *event_wait_list,
                          cl_event *event,
                          cl_int cmd) try {
   auto &q = obj(d_q);

   if (!q.device().svm_support())
      return CL_INVALID_OPERATION;

   if (svm_ptr == nullptr || pattern == nullptr ||
       !util_is_power_of_two_nonzero(pattern_size) ||
       pattern_size > 128 ||
       !ptr_is_aligned(svm_ptr, pattern_size) ||
       size % pattern_size)
      return CL_INVALID_VALUE;

   bool can_emulate = q.device().has_system_svm();
   auto deps = objs<wait_list_tag>(event_wait_list, num_events_in_wait_list);

   validate_common(q, deps);

   if (can_emulate) {
      auto hev = create<hard_event>(q, cmd, deps,
         [=](clover::event &) {
            void *ptr = svm_ptr;
            for (size_t s = size; s; s -= pattern_size) {
               memcpy(ptr, pattern, pattern_size);
               ptr = static_cast<uint8_t*>(ptr) + pattern_size;
            }
         });

      ret_object(event, hev);
      return CL_SUCCESS;
   }

   CLOVER_NOT_SUPPORTED_UNTIL("2.0");
   return CL_INVALID_VALUE;

} catch (error &e) {
   return e.get();
}

CLOVER_API cl_int
clEnqueueSVMMemFill(cl_command_queue d_q,
                    void *svm_ptr,
                    const void *pattern,
                    size_t pattern_size,
                    size_t size,
                    cl_uint num_events_in_wait_list,
                    const cl_event *event_wait_list,
                    cl_event *event) {

   return EnqueueSVMMemFill(d_q, svm_ptr, pattern, pattern_size,
                            size, num_events_in_wait_list, event_wait_list,
                            event, CL_COMMAND_SVM_MEMFILL);
}

cl_int
clover::EnqueueSVMMap(cl_command_queue d_q,
                      cl_bool blocking_map,
                      cl_map_flags map_flags,
                      void *svm_ptr,
                      size_t size,
                      cl_uint num_events_in_wait_list,
                      const cl_event *event_wait_list,
                      cl_event *event,
                      cl_int cmd) try {
   auto &q = obj(d_q);

   if (!q.device().svm_support())
      return CL_INVALID_OPERATION;

   if (svm_ptr == nullptr || size == 0)
      return CL_INVALID_VALUE;

   bool can_emulate = q.device().has_system_svm();
   auto deps = objs<wait_list_tag>(event_wait_list, num_events_in_wait_list);

   validate_common(q, deps);

   if (can_emulate) {
      auto hev = create<hard_event>(q, cmd, deps,
         [](clover::event &) { });

      ret_object(event, hev);
      return CL_SUCCESS;
   }

   CLOVER_NOT_SUPPORTED_UNTIL("2.0");
   return CL_INVALID_VALUE;

} catch (error &e) {
   return e.get();
}

CLOVER_API cl_int
clEnqueueSVMMap(cl_command_queue d_q,
                cl_bool blocking_map,
                cl_map_flags map_flags,
                void *svm_ptr,
                size_t size,
                cl_uint num_events_in_wait_list,
                const cl_event *event_wait_list,
                cl_event *event) {

   return EnqueueSVMMap(d_q, blocking_map, map_flags, svm_ptr, size,
                        num_events_in_wait_list, event_wait_list, event,
                        CL_COMMAND_SVM_MAP);
}

cl_int
clover::EnqueueSVMUnmap(cl_command_queue d_q,
                        void *svm_ptr,
                        cl_uint num_events_in_wait_list,
                        const cl_event *event_wait_list,
                        cl_event *event,
                        cl_int cmd) try {
   auto &q = obj(d_q);

   if (!q.device().svm_support())
      return CL_INVALID_OPERATION;

   if (svm_ptr == nullptr)
      return CL_INVALID_VALUE;

   bool can_emulate = q.device().has_system_svm();
   auto deps = objs<wait_list_tag>(event_wait_list, num_events_in_wait_list);

   validate_common(q, deps);

   if (can_emulate) {
      auto hev = create<hard_event>(q, cmd, deps,
         [](clover::event &) { });

      ret_object(event, hev);
      return CL_SUCCESS;
   }

   CLOVER_NOT_SUPPORTED_UNTIL("2.0");
   return CL_INVALID_VALUE;

} catch (error &e) {
   return e.get();
}

CLOVER_API cl_int
clEnqueueSVMUnmap(cl_command_queue d_q,
                  void *svm_ptr,
                  cl_uint num_events_in_wait_list,
                  const cl_event *event_wait_list,
                  cl_event *event) {

   return EnqueueSVMUnmap(d_q, svm_ptr, num_events_in_wait_list,
                          event_wait_list, event, CL_COMMAND_SVM_UNMAP);
}

CLOVER_API cl_int
clEnqueueSVMMigrateMem(cl_command_queue d_q,
                       cl_uint num_svm_pointers,
                       const void **svm_pointers,
                       const size_t *sizes,
                       const cl_mem_migration_flags flags,
                       cl_uint num_deps,
                       const cl_event *d_deps,
                       cl_event *rd_ev) try {
   auto &q = obj(d_q);
   auto deps = objs<wait_list_tag>(d_deps, num_deps);

   validate_common(q, deps);
   validate_mem_migration_flags(flags);

   if (!q.device().svm_support())
      return CL_INVALID_OPERATION;

   if (!num_svm_pointers || !svm_pointers)
      return CL_INVALID_VALUE;

   std::vector<size_t> sizes_copy(num_svm_pointers);
   std::vector<const void*>  ptrs(num_svm_pointers);

   for (unsigned i = 0; i < num_svm_pointers; ++i) {
      const void *ptr = svm_pointers[i];
      size_t size = sizes ? sizes[i] : 0;
      if (!ptr)
         return CL_INVALID_VALUE;

      auto p = q.context().find_svm_allocation(ptr);
      if (!p.first)
         return CL_INVALID_VALUE;

      std::ptrdiff_t pdiff = (uint8_t*)ptr - (uint8_t*)p.first;
      if (size && size + pdiff > p.second)
         return CL_INVALID_VALUE;

      sizes_copy[i] = size ? size : p.second;
      ptrs[i] = size ? svm_pointers[i] : p.first;
   }

   auto hev = create<hard_event>(
      q, CL_COMMAND_MIGRATE_MEM_OBJECTS, deps,
      [=, &q](event &) {
         q.svm_migrate(ptrs, sizes_copy, flags);
      });

   ret_object(rd_ev, hev);
   return CL_SUCCESS;

} catch (error &e) {
   return e.get();
}