From c0f03f6fc03e6d0b9e0c38f8a8b4f3b90bba84f2 Mon Sep 17 00:00:00 2001 From: Serge Martin Date: Sat, 9 May 2020 08:11:16 +0200 Subject: [PATCH] clover: implements clEnqueueMigrateMemObjects Reviewed-by: Francisco Jerez Part-of: --- src/gallium/frontends/clover/api/transfer.cpp | 71 ++++++++++++++++++++------ src/gallium/frontends/clover/core/kernel.cpp | 14 ++--- src/gallium/frontends/clover/core/memory.cpp | 57 ++++++++++++++++++--- src/gallium/frontends/clover/core/memory.hpp | 33 ++++++++++-- src/gallium/frontends/clover/core/resource.cpp | 5 +- src/gallium/frontends/clover/core/resource.hpp | 2 +- 6 files changed, 145 insertions(+), 37 deletions(-) diff --git a/src/gallium/frontends/clover/api/transfer.cpp b/src/gallium/frontends/clover/api/transfer.cpp index aa21b00..b2964b2 100644 --- a/src/gallium/frontends/clover/api/transfer.cpp +++ b/src/gallium/frontends/clover/api/transfer.cpp @@ -210,7 +210,7 @@ namespace { struct _map { _map(command_queue &q, image *img, cl_map_flags flags, vector_t offset, vector_t pitch, vector_t region) : - map(q, img->resource(q), flags, true, offset, region), + map(q, img->resource_in(q), flags, true, offset, region), pitch(map.pitch()) { } @@ -227,7 +227,7 @@ namespace { struct _map { _map(command_queue &q, buffer *mem, cl_map_flags flags, vector_t offset, vector_t pitch, vector_t region) : - map(q, mem->resource(q), flags, true, + map(q, mem->resource_in(q), flags, true, {{ dot(pitch, offset) }}, {{ size(pitch, region) }}), pitch(pitch) { } @@ -294,8 +294,8 @@ namespace { hard_copy_op(command_queue &q, T dst_obj, const vector_t &dst_orig, S src_obj, const vector_t &src_orig, const vector_t ®ion) { return [=, &q](event &) { - dst_obj->resource(q).copy(q, dst_orig, region, - src_obj->resource(q), src_orig); + dst_obj->resource_in(q).copy(q, dst_orig, region, + src_obj->resource_in(q), src_orig); }; } } @@ -480,7 +480,7 @@ clEnqueueFillBuffer(cl_command_queue d_queue, cl_mem d_mem, auto hev = create( q, CL_COMMAND_FILL_BUFFER, deps, [=, &q, &mem](event &) { - mem.resource(q).clear(q, offset, size, &data[0], data.size()); + mem.resource_in(q).clear(q, offset, size, &data[0], data.size()); }); ret_object(rd_ev, hev); @@ -755,7 +755,7 @@ clEnqueueMapBuffer(cl_command_queue d_q, cl_mem d_mem, cl_bool blocking, validate_object(q, mem, obj_origin, obj_pitch, region); validate_map_flags(mem, flags); - void *map = mem.resource(q).add_map(q, flags, blocking, obj_origin, region); + void *map = mem.resource_in(q).add_map(q, flags, blocking, obj_origin, region); auto hev = create(q, CL_COMMAND_MAP_BUFFER, deps); if (blocking) @@ -787,7 +787,7 @@ clEnqueueMapImage(cl_command_queue d_q, cl_mem d_mem, cl_bool blocking, validate_object(q, img, origin, region); validate_map_flags(img, flags); - void *map = img.resource(q).add_map(q, flags, blocking, origin, region); + void *map = img.resource_in(q).add_map(q, flags, blocking, origin, region); auto hev = create(q, CL_COMMAND_MAP_IMAGE, deps); if (blocking) @@ -815,7 +815,7 @@ clEnqueueUnmapMemObject(cl_command_queue d_q, cl_mem d_mem, void *ptr, auto hev = create( q, CL_COMMAND_UNMAP_MEM_OBJECT, deps, [=, &q, &mem](event &) { - mem.resource(q).del_map(ptr); + mem.resource_in(q).del_map(ptr); }); ret_object(rd_ev, hev); @@ -826,15 +826,54 @@ clEnqueueUnmapMemObject(cl_command_queue d_q, cl_mem d_mem, void *ptr, } CLOVER_API cl_int -clEnqueueMigrateMemObjects(cl_command_queue command_queue, - cl_uint num_mem_objects, - const cl_mem *mem_objects, +clEnqueueMigrateMemObjects(cl_command_queue d_q, + cl_uint num_mems, + const cl_mem *d_mems, cl_mem_migration_flags flags, - cl_uint num_events_in_wait_list, - const cl_event *event_wait_list, - cl_event *event) { - CLOVER_NOT_SUPPORTED_UNTIL("1.2"); - return CL_INVALID_VALUE; + cl_uint num_deps, + const cl_event *d_deps, + cl_event *rd_ev) try { + auto &q = obj(d_q); + auto mems = objs(d_mems, num_mems); + auto deps = objs(d_deps, num_deps); + + validate_common(q, deps); + + if (any_of([&](const memory_obj &m) { + return m.context() != q.context(); + }, mems)) + throw error(CL_INVALID_CONTEXT); + + if (flags & ~(CL_MIGRATE_MEM_OBJECT_HOST | + CL_MIGRATE_MEM_OBJECT_CONTENT_UNDEFINED)) + throw error(CL_INVALID_VALUE); + + auto hev = create( + q, CL_COMMAND_MIGRATE_MEM_OBJECTS, deps, + [=, &q](event &) { + for (auto &mem: mems) { + if (flags & CL_MIGRATE_MEM_OBJECT_HOST) { + if ((flags & CL_MIGRATE_MEM_OBJECT_CONTENT_UNDEFINED)) + mem.resource_out(q); + + // For flags == CL_MIGRATE_MEM_OBJECT_HOST only to be + // efficient we would need cl*ReadBuffer* to implement + // reading from host memory. + + } else { + if (flags & CL_MIGRATE_MEM_OBJECT_CONTENT_UNDEFINED) + mem.resource_undef(q); + else + mem.resource_in(q); + } + } + }); + + ret_object(rd_ev, hev); + return CL_SUCCESS;; + +} catch (error &e) { + return e.get(); } cl_int diff --git a/src/gallium/frontends/clover/core/kernel.cpp b/src/gallium/frontends/clover/core/kernel.cpp index 013d6f0..3000b7c 100644 --- a/src/gallium/frontends/clover/core/kernel.cpp +++ b/src/gallium/frontends/clover/core/kernel.cpp @@ -447,7 +447,7 @@ kernel::global_argument::bind(exec_context &ctx, align(ctx.input, marg.target_align); if (buf) { - const resource &r = buf->resource(*ctx.q); + const resource &r = buf->resource_in(*ctx.q); ctx.g_handles.push_back(ctx.input.size()); ctx.g_buffers.push_back(r.pipe); @@ -522,7 +522,7 @@ kernel::constant_argument::bind(exec_context &ctx, align(ctx.input, marg.target_align); if (buf) { - resource &r = buf->resource(*ctx.q); + resource &r = buf->resource_in(*ctx.q); auto v = bytes(ctx.resources.size() << 24 | r.offset[0]); extend(v, module::argument::zero_ext, marg.target_size); @@ -540,7 +540,7 @@ kernel::constant_argument::bind(exec_context &ctx, void kernel::constant_argument::unbind(exec_context &ctx) { if (buf) - buf->resource(*ctx.q).unbind_surface(*ctx.q, st); + buf->resource_in(*ctx.q).unbind_surface(*ctx.q, st); } void @@ -565,13 +565,13 @@ kernel::image_rd_argument::bind(exec_context &ctx, align(ctx.input, marg.target_align); insert(ctx.input, v); - st = img->resource(*ctx.q).bind_sampler_view(*ctx.q); + st = img->resource_in(*ctx.q).bind_sampler_view(*ctx.q); ctx.sviews.push_back(st); } void kernel::image_rd_argument::unbind(exec_context &ctx) { - img->resource(*ctx.q).unbind_sampler_view(*ctx.q, st); + img->resource_in(*ctx.q).unbind_sampler_view(*ctx.q, st); } void @@ -596,13 +596,13 @@ kernel::image_wr_argument::bind(exec_context &ctx, align(ctx.input, marg.target_align); insert(ctx.input, v); - st = img->resource(*ctx.q).bind_surface(*ctx.q, true); + st = img->resource_in(*ctx.q).bind_surface(*ctx.q, true); ctx.resources.push_back(st); } void kernel::image_wr_argument::unbind(exec_context &ctx) { - img->resource(*ctx.q).unbind_surface(*ctx.q, st); + img->resource_in(*ctx.q).unbind_surface(*ctx.q, st); } void diff --git a/src/gallium/frontends/clover/core/memory.cpp b/src/gallium/frontends/clover/core/memory.cpp index ed13d92..d20dbad 100644 --- a/src/gallium/frontends/clover/core/memory.cpp +++ b/src/gallium/frontends/clover/core/memory.cpp @@ -82,13 +82,27 @@ root_buffer::root_buffer(clover::context &ctx, cl_mem_flags flags, } resource & -root_buffer::resource(command_queue &q) { +root_buffer::resource_in(command_queue &q) { + const void *data_ptr = NULL; + if (flags() & (CL_MEM_USE_HOST_PTR | CL_MEM_COPY_HOST_PTR)) + data_ptr = !data.empty() ? data.data() : host_ptr(); + + return resource(q, data_ptr); +} + +resource & +root_buffer::resource_undef(command_queue &q) { + return resource(q, NULL); +} + +resource & +root_buffer::resource(command_queue &q, const void *data_ptr) { // Create a new resource if there's none for this device yet. if (!resources.count(&q.device())) { auto r = (!resources.empty() ? new root_resource(q.device(), *this, *resources.begin()->second) : - new root_resource(q.device(), *this, q, data)); + new root_resource(q.device(), *this, q, data_ptr)); resources.insert(std::make_pair(&q.device(), std::unique_ptr(r))); @@ -98,6 +112,11 @@ root_buffer::resource(command_queue &q) { return *resources.find(&q.device())->second; } +void +root_buffer::resource_out(command_queue &q) { + resources.erase(&q.device()); +} + sub_buffer::sub_buffer(root_buffer &parent, cl_mem_flags flags, size_t offset, size_t size) : buffer(parent.context(), flags, size, @@ -106,10 +125,10 @@ sub_buffer::sub_buffer(root_buffer &parent, cl_mem_flags flags, } resource & -sub_buffer::resource(command_queue &q) { +sub_buffer::resource_in(command_queue &q) { // Create a new resource if there's none for this device yet. if (!resources.count(&q.device())) { - auto r = new sub_resource(parent().resource(q), {{ offset() }}); + auto r = new sub_resource(parent().resource_in(q), {{ offset() }}); resources.insert(std::make_pair(&q.device(), std::unique_ptr(r))); @@ -118,6 +137,16 @@ sub_buffer::resource(command_queue &q) { return *resources.find(&q.device())->second; } +resource & +sub_buffer::resource_undef(command_queue &q) { + return resource_in(q); +} + +void +sub_buffer::resource_out(command_queue &q) { + resources.erase(&q.device()); +} + size_t sub_buffer::offset() const { return _offset; @@ -134,13 +163,24 @@ image::image(clover::context &ctx, cl_mem_flags flags, } resource & -image::resource(command_queue &q) { +image::resource_in(command_queue &q) { + const void *data_ptr = !data.empty() ? data.data() : NULL; + return resource(q, data_ptr); +} + +resource & +image::resource_undef(command_queue &q) { + return resource(q, NULL); +} + +resource & +image::resource(command_queue &q, const void *data_ptr) { // Create a new resource if there's none for this device yet. if (!resources.count(&q.device())) { auto r = (!resources.empty() ? new root_resource(q.device(), *this, *resources.begin()->second) : - new root_resource(q.device(), *this, q, data)); + new root_resource(q.device(), *this, q, data_ptr)); resources.insert(std::make_pair(&q.device(), std::unique_ptr(r))); @@ -150,6 +190,11 @@ image::resource(command_queue &q) { return *resources.find(&q.device())->second; } +void +image::resource_out(command_queue &q) { + resources.erase(&q.device()); +} + cl_image_format image::format() const { return _format; diff --git a/src/gallium/frontends/clover/core/memory.hpp b/src/gallium/frontends/clover/core/memory.hpp index bd6da6b..7c48110 100644 --- a/src/gallium/frontends/clover/core/memory.hpp +++ b/src/gallium/frontends/clover/core/memory.hpp @@ -49,7 +49,11 @@ namespace clover { operator==(const memory_obj &obj) const; virtual cl_mem_object_type type() const = 0; - virtual clover::resource &resource(command_queue &q) = 0; + virtual clover::resource & + resource_in(command_queue &q) = 0; + virtual clover::resource & + resource_undef(command_queue &q) = 0; + virtual void resource_out(command_queue &q) = 0; void destroy_notify(std::function f); cl_mem_flags flags() const; @@ -82,9 +86,17 @@ namespace clover { root_buffer(clover::context &ctx, cl_mem_flags flags, size_t size, void *host_ptr); - virtual clover::resource &resource(command_queue &q); + virtual clover::resource & + resource_in(command_queue &q); + virtual clover::resource & + resource_undef(command_queue &q); + virtual void + resource_out(command_queue &q); private: + clover::resource & + resource(command_queue &q, const void *data_ptr); + std::map> resources; }; @@ -94,7 +106,12 @@ namespace clover { sub_buffer(root_buffer &parent, cl_mem_flags flags, size_t offset, size_t size); - virtual clover::resource &resource(command_queue &q); + virtual clover::resource & + resource_in(command_queue &q); + virtual clover::resource & + resource_undef(command_queue &q); + virtual void + resource_out(command_queue &q); size_t offset() const; const intrusive_ref parent; @@ -114,7 +131,6 @@ namespace clover { void *host_ptr); public: - virtual clover::resource &resource(command_queue &q); cl_image_format format() const; size_t width() const; size_t height() const; @@ -122,8 +138,17 @@ namespace clover { size_t pixel_size() const; size_t row_pitch() const; size_t slice_pitch() const; + virtual clover::resource & + resource_in(command_queue &q); + virtual clover::resource & + resource_undef(command_queue &q); + virtual void + resource_out(command_queue &q); private: + clover::resource & + resource(command_queue &q, const void *data_ptr); + cl_image_format _format; size_t _width; size_t _height; diff --git a/src/gallium/frontends/clover/core/resource.cpp b/src/gallium/frontends/clover/core/resource.cpp index 2fbed90..f43163d 100644 --- a/src/gallium/frontends/clover/core/resource.cpp +++ b/src/gallium/frontends/clover/core/resource.cpp @@ -124,7 +124,7 @@ resource::unbind_surface(command_queue &q, pipe_surface *st) { } root_resource::root_resource(clover::device &dev, memory_obj &obj, - command_queue &q, const std::string &data) : + command_queue &q, const void *data_ptr) : resource(dev, obj) { pipe_resource info {}; @@ -161,8 +161,7 @@ root_resource::root_resource(clover::device &dev, memory_obj &obj, if (!pipe) throw error(CL_OUT_OF_RESOURCES); - if (obj.flags() & (CL_MEM_USE_HOST_PTR | CL_MEM_COPY_HOST_PTR)) { - const void *data_ptr = !data.empty() ? data.data() : obj.host_ptr(); + if (data_ptr) { box rect { {{ 0, 0, 0 }}, {{ info.width0, info.height0, info.depth0 }} }; unsigned cpp = util_format_get_blocksize(info.format); diff --git a/src/gallium/frontends/clover/core/resource.hpp b/src/gallium/frontends/clover/core/resource.hpp index 009e42f..51ecc53 100644 --- a/src/gallium/frontends/clover/core/resource.hpp +++ b/src/gallium/frontends/clover/core/resource.hpp @@ -89,7 +89,7 @@ namespace clover { class root_resource : public resource { public: root_resource(clover::device &dev, memory_obj &obj, - command_queue &q, const std::string &data); + command_queue &q, const void *data_ptr); root_resource(clover::device &dev, memory_obj &obj, root_resource &r); virtual ~root_resource(); }; -- 2.7.4