1 /*
2 * Copyright © 2022 Google, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 * SOFTWARE.
22 */
23
24 #include <assert.h>
25 #include <inttypes.h>
26 #include <pthread.h>
27
28 #include "util/libsync.h"
29 #include "util/os_file.h"
30
31 #include "drm/freedreno_ringbuffer_sp.h"
32 #include "virtio_priv.h"
33
34 static void
retire_execute(void * job,void * gdata,int thread_index)35 retire_execute(void *job, void *gdata, int thread_index)
36 {
37 struct fd_submit_sp *fd_submit = job;
38
39 MESA_TRACE_FUNC();
40
41 fd_fence_wait(fd_submit->out_fence);
42 }
43
44 static void
retire_cleanup(void * job,void * gdata,int thread_index)45 retire_cleanup(void *job, void *gdata, int thread_index)
46 {
47 struct fd_submit_sp *fd_submit = job;
48 fd_submit_del(&fd_submit->base);
49 }
50
51 static int
flush_submit_list(struct list_head * submit_list)52 flush_submit_list(struct list_head *submit_list)
53 {
54 struct fd_submit_sp *fd_submit = to_fd_submit_sp(last_submit(submit_list));
55 struct virtio_pipe *virtio_pipe = to_virtio_pipe(fd_submit->base.pipe);
56 struct fd_pipe *pipe = &virtio_pipe->base;
57 struct fd_device *dev = pipe->dev;
58
59 unsigned nr_cmds = 0;
60
61 MESA_TRACE_FUNC();
62
63 /* Determine the number of extra cmds's from deferred submits that
64 * we will be merging in:
65 */
66 foreach_submit (submit, submit_list) {
67 assert(submit->pipe == &virtio_pipe->base);
68 nr_cmds += to_fd_ringbuffer_sp(submit->primary)->u.nr_cmds;
69 }
70
71 /* TODO we can get rid of the extra copy into the req by just
72 * assuming the max amount that nr->bos will grow is by the
73 * nr_cmds, and just over-allocate a bit.
74 */
75
76 struct drm_msm_gem_submit_cmd cmds[nr_cmds];
77
78 unsigned cmd_idx = 0;
79
80 /* Build up the table of cmds, and for all but the last submit in the
81 * list, merge their bo tables into the last submit.
82 */
83 foreach_submit_safe (submit, submit_list) {
84 struct fd_ringbuffer_sp *deferred_primary =
85 to_fd_ringbuffer_sp(submit->primary);
86
87 for (unsigned i = 0; i < deferred_primary->u.nr_cmds; i++) {
88 struct fd_bo *ring_bo = deferred_primary->u.cmds[i].ring_bo;
89 cmds[cmd_idx].type = MSM_SUBMIT_CMD_BUF;
90 cmds[cmd_idx].submit_idx = fd_submit_append_bo(fd_submit, ring_bo);
91 cmds[cmd_idx].submit_offset = submit_offset(ring_bo, deferred_primary->offset);
92 cmds[cmd_idx].size = deferred_primary->u.cmds[i].size;
93 cmds[cmd_idx].pad = 0;
94 cmds[cmd_idx].nr_relocs = 0;
95
96 cmd_idx++;
97 }
98
99 /* We are merging all the submits in the list into the last submit,
100 * so the remainder of the loop body doesn't apply to the last submit
101 */
102 if (submit == last_submit(submit_list)) {
103 DEBUG_MSG("merged %u submits", cmd_idx);
104 break;
105 }
106
107 struct fd_submit_sp *fd_deferred_submit = to_fd_submit_sp(submit);
108 for (unsigned i = 0; i < fd_deferred_submit->nr_bos; i++) {
109 /* Note: if bo is used in both the current submit and the deferred
110 * submit being merged, we expect to hit the fast-path as we add it
111 * to the current submit:
112 */
113 fd_submit_append_bo(fd_submit, fd_deferred_submit->bos[i]);
114 }
115
116 /* Now that the cmds/bos have been transfered over to the current submit,
117 * we can remove the deferred submit from the list and drop it's reference
118 */
119 list_del(&submit->node);
120 fd_submit_del(submit);
121 }
122
123 /* Needs to be after get_cmd() as that could create bos/cmds table:
124 *
125 * NOTE allocate on-stack in the common case, but with an upper-
126 * bound to limit on-stack allocation to 4k:
127 */
128 const unsigned bo_limit = 4096 / sizeof(struct drm_msm_gem_submit_bo);
129 bool bos_on_stack = fd_submit->nr_bos < bo_limit;
130 struct drm_msm_gem_submit_bo
131 _submit_bos[bos_on_stack ? fd_submit->nr_bos : 0];
132 struct drm_msm_gem_submit_bo *submit_bos;
133 uint32_t _guest_handles[bos_on_stack ? fd_submit->nr_bos : 0];
134 uint32_t *guest_handles;
135 if (bos_on_stack) {
136 submit_bos = _submit_bos;
137 guest_handles = _guest_handles;
138 } else {
139 submit_bos = malloc(fd_submit->nr_bos * sizeof(submit_bos[0]));
140 guest_handles = malloc(fd_submit->nr_bos * sizeof(guest_handles[0]));
141 }
142
143 uint32_t nr_guest_handles = 0;
144 for (unsigned i = 0; i < fd_submit->nr_bos; i++) {
145 struct virtio_bo *virtio_bo = to_virtio_bo(fd_submit->bos[i]);
146
147 if (virtio_bo->base.alloc_flags & FD_BO_SHARED)
148 guest_handles[nr_guest_handles++] = virtio_bo->base.handle;
149
150 submit_bos[i].flags = fd_submit->bos[i]->reloc_flags;
151 submit_bos[i].handle = virtio_bo->res_id;
152 submit_bos[i].presumed = 0;
153 }
154
155 if (virtio_pipe->next_submit_fence <= 0)
156 virtio_pipe->next_submit_fence = 1;
157
158 uint32_t kfence = virtio_pipe->next_submit_fence++;
159
160 /* TODO avoid extra memcpy, and populate bo's and cmds directly
161 * into the req msg
162 */
163 unsigned bos_len = fd_submit->nr_bos * sizeof(struct drm_msm_gem_submit_bo);
164 unsigned cmd_len = nr_cmds * sizeof(struct drm_msm_gem_submit_cmd);
165 unsigned req_len = sizeof(struct msm_ccmd_gem_submit_req) + bos_len + cmd_len;
166 struct msm_ccmd_gem_submit_req *req = malloc(req_len);
167
168 req->hdr = MSM_CCMD(GEM_SUBMIT, req_len);
169 req->flags = virtio_pipe->pipe;
170 req->queue_id = virtio_pipe->queue_id;
171 req->nr_bos = fd_submit->nr_bos;
172 req->nr_cmds = nr_cmds;
173 req->fence = kfence;
174
175 memcpy(req->payload, submit_bos, bos_len);
176 memcpy(req->payload + bos_len, cmds, cmd_len);
177
178 struct fd_fence *out_fence = fd_submit->out_fence;
179
180 out_fence->kfence = kfence;
181
182 /* Even if gallium driver hasn't requested a fence-fd, request one.
183 * This way, if we have to block waiting for the fence, we can do
184 * it in the guest, rather than in the single-threaded host.
185 */
186 out_fence->use_fence_fd = true;
187
188 if (pipe->no_implicit_sync) {
189 req->flags |= MSM_SUBMIT_NO_IMPLICIT;
190 nr_guest_handles = 0;
191 }
192
193 struct vdrm_execbuf_params p = {
194 .req = &req->hdr,
195 .handles = guest_handles,
196 .num_handles = nr_guest_handles,
197 .has_in_fence_fd = !!(fd_submit->in_fence_fd != -1),
198 .needs_out_fence_fd = true,
199 .fence_fd = fd_submit->in_fence_fd,
200 .ring_idx = virtio_pipe->ring_idx,
201 };
202 vdrm_execbuf(to_virtio_device(dev)->vdrm, &p);
203
204 out_fence->fence_fd = p.fence_fd;
205
206 free(req);
207
208 if (!bos_on_stack) {
209 free(submit_bos);
210 free(guest_handles);
211 }
212
213 if (fd_submit->in_fence_fd != -1)
214 close(fd_submit->in_fence_fd);
215
216 fd_submit_ref(&fd_submit->base);
217
218 util_queue_fence_init(&fd_submit->retire_fence);
219
220 util_queue_add_job(&virtio_pipe->retire_queue,
221 fd_submit, &fd_submit->retire_fence,
222 retire_execute,
223 retire_cleanup,
224 0);
225
226 return 0;
227 }
228
229 struct fd_submit *
virtio_submit_new(struct fd_pipe * pipe)230 virtio_submit_new(struct fd_pipe *pipe)
231 {
232 /* We don't do any translation from internal FD_RELOC flags to MSM flags. */
233 STATIC_ASSERT(FD_RELOC_READ == MSM_SUBMIT_BO_READ);
234 STATIC_ASSERT(FD_RELOC_WRITE == MSM_SUBMIT_BO_WRITE);
235 STATIC_ASSERT(FD_RELOC_DUMP == MSM_SUBMIT_BO_DUMP);
236
237 return fd_submit_sp_new(pipe, flush_submit_list);
238 }
239