1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2019 Intel Corporation
4 */
5
6 #include "gem/i915_gem_lmem.h"
7 #include "gem/i915_gem_object.h"
8
9 #include "i915_drv.h"
10 #include "i915_vma.h"
11 #include "intel_engine.h"
12 #include "intel_gpu_commands.h"
13 #include "intel_ring.h"
14 #include "intel_timeline.h"
15
intel_ring_update_space(struct intel_ring * ring)16 unsigned int intel_ring_update_space(struct intel_ring *ring)
17 {
18 unsigned int space;
19
20 space = __intel_ring_space(ring->head, ring->emit, ring->size);
21
22 ring->space = space;
23 return space;
24 }
25
__intel_ring_pin(struct intel_ring * ring)26 void __intel_ring_pin(struct intel_ring *ring)
27 {
28 GEM_BUG_ON(!atomic_read(&ring->pin_count));
29 atomic_inc(&ring->pin_count);
30 }
31
intel_ring_pin(struct intel_ring * ring,struct i915_gem_ww_ctx * ww)32 int intel_ring_pin(struct intel_ring *ring, struct i915_gem_ww_ctx *ww)
33 {
34 struct i915_vma *vma = ring->vma;
35 unsigned int flags;
36 void *addr;
37 int ret;
38
39 if (atomic_fetch_inc(&ring->pin_count))
40 return 0;
41
42 /* Ring wraparound at offset 0 sometimes hangs. No idea why. */
43 flags = PIN_OFFSET_BIAS | i915_ggtt_pin_bias(vma);
44
45 if (i915_gem_object_is_stolen(vma->obj))
46 flags |= PIN_MAPPABLE;
47 else
48 flags |= PIN_HIGH;
49
50 ret = i915_ggtt_pin(vma, ww, 0, flags);
51 if (unlikely(ret))
52 goto err_unpin;
53
54 if (i915_vma_is_map_and_fenceable(vma)) {
55 addr = (void __force *)i915_vma_pin_iomap(vma);
56 } else {
57 int type = i915_coherent_map_type(vma->vm->i915, vma->obj, false);
58
59 addr = i915_gem_object_pin_map(vma->obj, type);
60 }
61
62 if (IS_ERR(addr)) {
63 ret = PTR_ERR(addr);
64 goto err_ring;
65 }
66
67 i915_vma_make_unshrinkable(vma);
68
69 /* Discard any unused bytes beyond that submitted to hw. */
70 intel_ring_reset(ring, ring->emit);
71
72 ring->vaddr = addr;
73 return 0;
74
75 err_ring:
76 i915_vma_unpin(vma);
77 err_unpin:
78 atomic_dec(&ring->pin_count);
79 return ret;
80 }
81
intel_ring_reset(struct intel_ring * ring,u32 tail)82 void intel_ring_reset(struct intel_ring *ring, u32 tail)
83 {
84 tail = intel_ring_wrap(ring, tail);
85 ring->tail = tail;
86 ring->head = tail;
87 ring->emit = tail;
88 intel_ring_update_space(ring);
89 }
90
intel_ring_unpin(struct intel_ring * ring)91 void intel_ring_unpin(struct intel_ring *ring)
92 {
93 struct i915_vma *vma = ring->vma;
94
95 if (!atomic_dec_and_test(&ring->pin_count))
96 return;
97
98 i915_vma_unset_ggtt_write(vma);
99 if (i915_vma_is_map_and_fenceable(vma))
100 i915_vma_unpin_iomap(vma);
101 else
102 i915_gem_object_unpin_map(vma->obj);
103
104 i915_vma_make_purgeable(vma);
105 i915_vma_unpin(vma);
106 }
107
create_ring_vma(struct i915_ggtt * ggtt,int size)108 static struct i915_vma *create_ring_vma(struct i915_ggtt *ggtt, int size)
109 {
110 struct i915_address_space *vm = &ggtt->vm;
111 struct drm_i915_private *i915 = vm->i915;
112 struct drm_i915_gem_object *obj;
113 struct i915_vma *vma;
114
115 obj = i915_gem_object_create_lmem(i915, size, I915_BO_ALLOC_VOLATILE);
116 if (IS_ERR(obj) && i915_ggtt_has_aperture(ggtt) && !HAS_LLC(i915))
117 obj = i915_gem_object_create_stolen(i915, size);
118 if (IS_ERR(obj))
119 obj = i915_gem_object_create_internal(i915, size);
120 if (IS_ERR(obj))
121 return ERR_CAST(obj);
122
123 /*
124 * Mark ring buffers as read-only from GPU side (so no stray overwrites)
125 * if supported by the platform's GGTT.
126 */
127 if (vm->has_read_only)
128 i915_gem_object_set_readonly(obj);
129
130 vma = i915_vma_instance(obj, vm, NULL);
131 if (IS_ERR(vma))
132 goto err;
133
134 return vma;
135
136 err:
137 i915_gem_object_put(obj);
138 return vma;
139 }
140
141 struct intel_ring *
intel_engine_create_ring(struct intel_engine_cs * engine,int size)142 intel_engine_create_ring(struct intel_engine_cs *engine, int size)
143 {
144 struct drm_i915_private *i915 = engine->i915;
145 struct intel_ring *ring;
146 struct i915_vma *vma;
147
148 GEM_BUG_ON(!is_power_of_2(size));
149 GEM_BUG_ON(RING_CTL_SIZE(size) & ~RING_NR_PAGES);
150
151 ring = kzalloc(sizeof(*ring), GFP_KERNEL);
152 if (!ring)
153 return ERR_PTR(-ENOMEM);
154
155 kref_init(&ring->ref);
156 ring->size = size;
157 ring->wrap = BITS_PER_TYPE(ring->size) - ilog2(size);
158
159 /*
160 * Workaround an erratum on the i830 which causes a hang if
161 * the TAIL pointer points to within the last 2 cachelines
162 * of the buffer.
163 */
164 ring->effective_size = size;
165 if (IS_I830(i915) || IS_I845G(i915))
166 ring->effective_size -= 2 * CACHELINE_BYTES;
167
168 intel_ring_update_space(ring);
169
170 vma = create_ring_vma(engine->gt->ggtt, size);
171 if (IS_ERR(vma)) {
172 kfree(ring);
173 return ERR_CAST(vma);
174 }
175 ring->vma = vma;
176
177 return ring;
178 }
179
intel_ring_free(struct kref * ref)180 void intel_ring_free(struct kref *ref)
181 {
182 struct intel_ring *ring = container_of(ref, typeof(*ring), ref);
183
184 i915_vma_put(ring->vma);
185 kfree(ring);
186 }
187
188 static noinline int
wait_for_space(struct intel_ring * ring,struct intel_timeline * tl,unsigned int bytes)189 wait_for_space(struct intel_ring *ring,
190 struct intel_timeline *tl,
191 unsigned int bytes)
192 {
193 struct i915_request *target;
194 long timeout;
195
196 if (intel_ring_update_space(ring) >= bytes)
197 return 0;
198
199 GEM_BUG_ON(list_empty(&tl->requests));
200 list_for_each_entry(target, &tl->requests, link) {
201 if (target->ring != ring)
202 continue;
203
204 /* Would completion of this request free enough space? */
205 if (bytes <= __intel_ring_space(target->postfix,
206 ring->emit, ring->size))
207 break;
208 }
209
210 if (GEM_WARN_ON(&target->link == &tl->requests))
211 return -ENOSPC;
212
213 timeout = i915_request_wait(target,
214 I915_WAIT_INTERRUPTIBLE,
215 MAX_SCHEDULE_TIMEOUT);
216 if (timeout < 0)
217 return timeout;
218
219 i915_request_retire_upto(target);
220
221 intel_ring_update_space(ring);
222 GEM_BUG_ON(ring->space < bytes);
223 return 0;
224 }
225
intel_ring_begin(struct i915_request * rq,unsigned int num_dwords)226 u32 *intel_ring_begin(struct i915_request *rq, unsigned int num_dwords)
227 {
228 struct intel_ring *ring = rq->ring;
229 const unsigned int remain_usable = ring->effective_size - ring->emit;
230 const unsigned int bytes = num_dwords * sizeof(u32);
231 unsigned int need_wrap = 0;
232 unsigned int total_bytes;
233 u32 *cs;
234
235 /* Packets must be qword aligned. */
236 GEM_BUG_ON(num_dwords & 1);
237
238 total_bytes = bytes + rq->reserved_space;
239 GEM_BUG_ON(total_bytes > ring->effective_size);
240
241 if (unlikely(total_bytes > remain_usable)) {
242 const int remain_actual = ring->size - ring->emit;
243
244 if (bytes > remain_usable) {
245 /*
246 * Not enough space for the basic request. So need to
247 * flush out the remainder and then wait for
248 * base + reserved.
249 */
250 total_bytes += remain_actual;
251 need_wrap = remain_actual | 1;
252 } else {
253 /*
254 * The base request will fit but the reserved space
255 * falls off the end. So we don't need an immediate
256 * wrap and only need to effectively wait for the
257 * reserved size from the start of ringbuffer.
258 */
259 total_bytes = rq->reserved_space + remain_actual;
260 }
261 }
262
263 if (unlikely(total_bytes > ring->space)) {
264 int ret;
265
266 /*
267 * Space is reserved in the ringbuffer for finalising the
268 * request, as that cannot be allowed to fail. During request
269 * finalisation, reserved_space is set to 0 to stop the
270 * overallocation and the assumption is that then we never need
271 * to wait (which has the risk of failing with EINTR).
272 *
273 * See also i915_request_alloc() and i915_request_add().
274 */
275 GEM_BUG_ON(!rq->reserved_space);
276
277 ret = wait_for_space(ring,
278 i915_request_timeline(rq),
279 total_bytes);
280 if (unlikely(ret))
281 return ERR_PTR(ret);
282 }
283
284 if (unlikely(need_wrap)) {
285 need_wrap &= ~1;
286 GEM_BUG_ON(need_wrap > ring->space);
287 GEM_BUG_ON(ring->emit + need_wrap > ring->size);
288 GEM_BUG_ON(!IS_ALIGNED(need_wrap, sizeof(u64)));
289
290 /* Fill the tail with MI_NOOP */
291 memset64(ring->vaddr + ring->emit, 0, need_wrap / sizeof(u64));
292 ring->space -= need_wrap;
293 ring->emit = 0;
294 }
295
296 GEM_BUG_ON(ring->emit > ring->size - bytes);
297 GEM_BUG_ON(ring->space < bytes);
298 cs = ring->vaddr + ring->emit;
299 GEM_DEBUG_EXEC(memset32(cs, POISON_INUSE, bytes / sizeof(*cs)));
300 ring->emit += bytes;
301 ring->space -= bytes;
302
303 return cs;
304 }
305
306 /* Align the ring tail to a cacheline boundary */
intel_ring_cacheline_align(struct i915_request * rq)307 int intel_ring_cacheline_align(struct i915_request *rq)
308 {
309 int num_dwords;
310 void *cs;
311
312 num_dwords = (rq->ring->emit & (CACHELINE_BYTES - 1)) / sizeof(u32);
313 if (num_dwords == 0)
314 return 0;
315
316 num_dwords = CACHELINE_DWORDS - num_dwords;
317 GEM_BUG_ON(num_dwords & 1);
318
319 cs = intel_ring_begin(rq, num_dwords);
320 if (IS_ERR(cs))
321 return PTR_ERR(cs);
322
323 memset64(cs, (u64)MI_NOOP << 32 | MI_NOOP, num_dwords / 2);
324 intel_ring_advance(rq, cs + num_dwords);
325
326 GEM_BUG_ON(rq->ring->emit & (CACHELINE_BYTES - 1));
327 return 0;
328 }
329
330 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
331 #include "selftest_ring.c"
332 #endif
333