1 /*
2 * SPDX-License-Identifier: MIT
3 *
4 * Copyright © 2018 Intel Corporation
5 */
6
7 #include <linux/kref.h>
8
9 #include "gem/i915_gem_pm.h"
10 #include "gt/intel_gt.h"
11
12 #include "i915_selftest.h"
13
14 #include "igt_flush_test.h"
15 #include "lib_sw_fence.h"
16
17 struct live_active {
18 struct i915_active base;
19 struct kref ref;
20 bool retired;
21 };
22
__live_get(struct live_active * active)23 static void __live_get(struct live_active *active)
24 {
25 kref_get(&active->ref);
26 }
27
__live_free(struct live_active * active)28 static void __live_free(struct live_active *active)
29 {
30 i915_active_fini(&active->base);
31 kfree(active);
32 }
33
__live_release(struct kref * ref)34 static void __live_release(struct kref *ref)
35 {
36 struct live_active *active = container_of(ref, typeof(*active), ref);
37
38 __live_free(active);
39 }
40
__live_put(struct live_active * active)41 static void __live_put(struct live_active *active)
42 {
43 kref_put(&active->ref, __live_release);
44 }
45
__live_active(struct i915_active * base)46 static int __live_active(struct i915_active *base)
47 {
48 struct live_active *active = container_of(base, typeof(*active), base);
49
50 __live_get(active);
51 return 0;
52 }
53
__live_retire(struct i915_active * base)54 static void __live_retire(struct i915_active *base)
55 {
56 struct live_active *active = container_of(base, typeof(*active), base);
57
58 active->retired = true;
59 __live_put(active);
60 }
61
__live_alloc(struct drm_i915_private * i915)62 static struct live_active *__live_alloc(struct drm_i915_private *i915)
63 {
64 struct live_active *active;
65
66 active = kzalloc(sizeof(*active), GFP_KERNEL);
67 if (!active)
68 return NULL;
69
70 kref_init(&active->ref);
71 i915_active_init(&active->base, __live_active, __live_retire);
72
73 return active;
74 }
75
76 static struct live_active *
__live_active_setup(struct drm_i915_private * i915)77 __live_active_setup(struct drm_i915_private *i915)
78 {
79 struct intel_engine_cs *engine;
80 struct i915_sw_fence *submit;
81 struct live_active *active;
82 unsigned int count = 0;
83 int err = 0;
84
85 active = __live_alloc(i915);
86 if (!active)
87 return ERR_PTR(-ENOMEM);
88
89 submit = heap_fence_create(GFP_KERNEL);
90 if (!submit) {
91 kfree(active);
92 return ERR_PTR(-ENOMEM);
93 }
94
95 err = i915_active_acquire(&active->base);
96 if (err)
97 goto out;
98
99 for_each_uabi_engine(engine, i915) {
100 struct i915_request *rq;
101
102 rq = intel_engine_create_kernel_request(engine);
103 if (IS_ERR(rq)) {
104 err = PTR_ERR(rq);
105 break;
106 }
107
108 err = i915_sw_fence_await_sw_fence_gfp(&rq->submit,
109 submit,
110 GFP_KERNEL);
111 if (err >= 0)
112 err = i915_active_add_request(&active->base, rq);
113 i915_request_add(rq);
114 if (err) {
115 pr_err("Failed to track active ref!\n");
116 break;
117 }
118
119 count++;
120 }
121
122 i915_active_release(&active->base);
123 if (READ_ONCE(active->retired) && count) {
124 pr_err("i915_active retired before submission!\n");
125 err = -EINVAL;
126 }
127 if (atomic_read(&active->base.count) != count) {
128 pr_err("i915_active not tracking all requests, found %d, expected %d\n",
129 atomic_read(&active->base.count), count);
130 err = -EINVAL;
131 }
132
133 out:
134 i915_sw_fence_commit(submit);
135 heap_fence_put(submit);
136 if (err) {
137 __live_put(active);
138 active = ERR_PTR(err);
139 }
140
141 return active;
142 }
143
live_active_wait(void * arg)144 static int live_active_wait(void *arg)
145 {
146 struct drm_i915_private *i915 = arg;
147 struct live_active *active;
148 int err = 0;
149
150 /* Check that we get a callback when requests retire upon waiting */
151
152 active = __live_active_setup(i915);
153 if (IS_ERR(active))
154 return PTR_ERR(active);
155
156 __i915_active_wait(&active->base, TASK_UNINTERRUPTIBLE);
157 if (!READ_ONCE(active->retired)) {
158 struct drm_printer p = drm_err_printer(__func__);
159
160 pr_err("i915_active not retired after waiting!\n");
161 i915_active_print(&active->base, &p);
162
163 err = -EINVAL;
164 }
165
166 __live_put(active);
167
168 if (igt_flush_test(i915))
169 err = -EIO;
170
171 return err;
172 }
173
live_active_retire(void * arg)174 static int live_active_retire(void *arg)
175 {
176 struct drm_i915_private *i915 = arg;
177 struct live_active *active;
178 int err = 0;
179
180 /* Check that we get a callback when requests are indirectly retired */
181
182 active = __live_active_setup(i915);
183 if (IS_ERR(active))
184 return PTR_ERR(active);
185
186 /* waits for & retires all requests */
187 if (igt_flush_test(i915))
188 err = -EIO;
189
190 if (!READ_ONCE(active->retired)) {
191 struct drm_printer p = drm_err_printer(__func__);
192
193 pr_err("i915_active not retired after flushing!\n");
194 i915_active_print(&active->base, &p);
195
196 err = -EINVAL;
197 }
198
199 __live_put(active);
200
201 return err;
202 }
203
live_active_barrier(void * arg)204 static int live_active_barrier(void *arg)
205 {
206 struct drm_i915_private *i915 = arg;
207 struct intel_engine_cs *engine;
208 struct live_active *active;
209 int err = 0;
210
211 /* Check that we get a callback when requests retire upon waiting */
212
213 active = __live_alloc(i915);
214 if (!active)
215 return -ENOMEM;
216
217 err = i915_active_acquire(&active->base);
218 if (err)
219 goto out;
220
221 for_each_uabi_engine(engine, i915) {
222 err = i915_active_acquire_preallocate_barrier(&active->base,
223 engine);
224 if (err)
225 break;
226
227 i915_active_acquire_barrier(&active->base);
228 }
229
230 i915_active_release(&active->base);
231 if (err)
232 goto out;
233
234 __i915_active_wait(&active->base, TASK_UNINTERRUPTIBLE);
235 if (!READ_ONCE(active->retired)) {
236 pr_err("i915_active not retired after flushing barriers!\n");
237 err = -EINVAL;
238 }
239
240 out:
241 __live_put(active);
242
243 if (igt_flush_test(i915))
244 err = -EIO;
245
246 return err;
247 }
248
i915_active_live_selftests(struct drm_i915_private * i915)249 int i915_active_live_selftests(struct drm_i915_private *i915)
250 {
251 static const struct i915_subtest tests[] = {
252 SUBTEST(live_active_wait),
253 SUBTEST(live_active_retire),
254 SUBTEST(live_active_barrier),
255 };
256
257 if (intel_gt_is_wedged(&i915->gt))
258 return 0;
259
260 return i915_subtests(tests, i915);
261 }
262
node_to_barrier(struct active_node * it)263 static struct intel_engine_cs *node_to_barrier(struct active_node *it)
264 {
265 struct intel_engine_cs *engine;
266
267 if (!is_barrier(&it->base))
268 return NULL;
269
270 engine = __barrier_to_engine(it);
271 smp_rmb(); /* serialise with add_active_barriers */
272 if (!is_barrier(&it->base))
273 return NULL;
274
275 return engine;
276 }
277
i915_active_print(struct i915_active * ref,struct drm_printer * m)278 void i915_active_print(struct i915_active *ref, struct drm_printer *m)
279 {
280 drm_printf(m, "active %ps:%ps\n", ref->active, ref->retire);
281 drm_printf(m, "\tcount: %d\n", atomic_read(&ref->count));
282 drm_printf(m, "\tpreallocated barriers? %s\n",
283 yesno(!llist_empty(&ref->preallocated_barriers)));
284
285 if (i915_active_acquire_if_busy(ref)) {
286 struct active_node *it, *n;
287
288 rbtree_postorder_for_each_entry_safe(it, n, &ref->tree, node) {
289 struct intel_engine_cs *engine;
290
291 engine = node_to_barrier(it);
292 if (engine) {
293 drm_printf(m, "\tbarrier: %s\n", engine->name);
294 continue;
295 }
296
297 if (i915_active_fence_isset(&it->base)) {
298 drm_printf(m,
299 "\ttimeline: %llx\n", it->timeline);
300 continue;
301 }
302 }
303
304 i915_active_release(ref);
305 }
306 }
307
spin_unlock_wait(spinlock_t * lock)308 static void spin_unlock_wait(spinlock_t *lock)
309 {
310 spin_lock_irq(lock);
311 spin_unlock_irq(lock);
312 }
313
active_flush(struct i915_active * ref,struct i915_active_fence * active)314 static void active_flush(struct i915_active *ref,
315 struct i915_active_fence *active)
316 {
317 struct dma_fence *fence;
318
319 fence = xchg(__active_fence_slot(active), NULL);
320 if (!fence)
321 return;
322
323 spin_lock_irq(fence->lock);
324 __list_del_entry(&active->cb.node);
325 spin_unlock_irq(fence->lock); /* serialise with fence->cb_list */
326 atomic_dec(&ref->count);
327
328 GEM_BUG_ON(!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags));
329 }
330
i915_active_unlock_wait(struct i915_active * ref)331 void i915_active_unlock_wait(struct i915_active *ref)
332 {
333 if (i915_active_acquire_if_busy(ref)) {
334 struct active_node *it, *n;
335
336 /* Wait for all active callbacks */
337 rcu_read_lock();
338 active_flush(ref, &ref->excl);
339 rbtree_postorder_for_each_entry_safe(it, n, &ref->tree, node)
340 active_flush(ref, &it->base);
341 rcu_read_unlock();
342
343 i915_active_release(ref);
344 }
345
346 /* And wait for the retire callback */
347 spin_unlock_wait(&ref->tree_lock);
348
349 /* ... which may have been on a thread instead */
350 flush_work(&ref->work);
351 }
352