1 /*
2 * Copyright © 2016 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 */
24
25 #include "../i915_selftest.h"
26
27 #include "mock_gem_device.h"
28 #include "huge_gem_object.h"
29
igt_gem_object(void * arg)30 static int igt_gem_object(void *arg)
31 {
32 struct drm_i915_private *i915 = arg;
33 struct drm_i915_gem_object *obj;
34 int err = -ENOMEM;
35
36 /* Basic test to ensure we can create an object */
37
38 obj = i915_gem_object_create(i915, PAGE_SIZE);
39 if (IS_ERR(obj)) {
40 err = PTR_ERR(obj);
41 pr_err("i915_gem_object_create failed, err=%d\n", err);
42 goto out;
43 }
44
45 err = 0;
46 i915_gem_object_put(obj);
47 out:
48 return err;
49 }
50
igt_phys_object(void * arg)51 static int igt_phys_object(void *arg)
52 {
53 struct drm_i915_private *i915 = arg;
54 struct drm_i915_gem_object *obj;
55 int err;
56
57 /* Create an object and bind it to a contiguous set of physical pages,
58 * i.e. exercise the i915_gem_object_phys API.
59 */
60
61 obj = i915_gem_object_create(i915, PAGE_SIZE);
62 if (IS_ERR(obj)) {
63 err = PTR_ERR(obj);
64 pr_err("i915_gem_object_create failed, err=%d\n", err);
65 goto out;
66 }
67
68 mutex_lock(&i915->drm.struct_mutex);
69 err = i915_gem_object_attach_phys(obj, PAGE_SIZE);
70 mutex_unlock(&i915->drm.struct_mutex);
71 if (err) {
72 pr_err("i915_gem_object_attach_phys failed, err=%d\n", err);
73 goto out_obj;
74 }
75
76 if (obj->ops != &i915_gem_phys_ops) {
77 pr_err("i915_gem_object_attach_phys did not create a phys object\n");
78 err = -EINVAL;
79 goto out_obj;
80 }
81
82 if (!atomic_read(&obj->mm.pages_pin_count)) {
83 pr_err("i915_gem_object_attach_phys did not pin its phys pages\n");
84 err = -EINVAL;
85 goto out_obj;
86 }
87
88 /* Make the object dirty so that put_pages must do copy back the data */
89 mutex_lock(&i915->drm.struct_mutex);
90 err = i915_gem_object_set_to_gtt_domain(obj, true);
91 mutex_unlock(&i915->drm.struct_mutex);
92 if (err) {
93 pr_err("i915_gem_object_set_to_gtt_domain failed with err=%d\n",
94 err);
95 goto out_obj;
96 }
97
98 out_obj:
99 i915_gem_object_put(obj);
100 out:
101 return err;
102 }
103
igt_gem_huge(void * arg)104 static int igt_gem_huge(void *arg)
105 {
106 const unsigned int nreal = 509; /* just to be awkward */
107 struct drm_i915_private *i915 = arg;
108 struct drm_i915_gem_object *obj;
109 unsigned int n;
110 int err;
111
112 /* Basic sanitycheck of our huge fake object allocation */
113
114 obj = huge_gem_object(i915,
115 nreal * PAGE_SIZE,
116 i915->ggtt.vm.total + PAGE_SIZE);
117 if (IS_ERR(obj))
118 return PTR_ERR(obj);
119
120 err = i915_gem_object_pin_pages(obj);
121 if (err) {
122 pr_err("Failed to allocate %u pages (%lu total), err=%d\n",
123 nreal, obj->base.size / PAGE_SIZE, err);
124 goto out;
125 }
126
127 for (n = 0; n < obj->base.size / PAGE_SIZE; n++) {
128 if (i915_gem_object_get_page(obj, n) !=
129 i915_gem_object_get_page(obj, n % nreal)) {
130 pr_err("Page lookup mismatch at index %u [%u]\n",
131 n, n % nreal);
132 err = -EINVAL;
133 goto out_unpin;
134 }
135 }
136
137 out_unpin:
138 i915_gem_object_unpin_pages(obj);
139 out:
140 i915_gem_object_put(obj);
141 return err;
142 }
143
144 struct tile {
145 unsigned int width;
146 unsigned int height;
147 unsigned int stride;
148 unsigned int size;
149 unsigned int tiling;
150 unsigned int swizzle;
151 };
152
swizzle_bit(unsigned int bit,u64 offset)153 static u64 swizzle_bit(unsigned int bit, u64 offset)
154 {
155 return (offset & BIT_ULL(bit)) >> (bit - 6);
156 }
157
tiled_offset(const struct tile * tile,u64 v)158 static u64 tiled_offset(const struct tile *tile, u64 v)
159 {
160 u64 x, y;
161
162 if (tile->tiling == I915_TILING_NONE)
163 return v;
164
165 y = div64_u64_rem(v, tile->stride, &x);
166 v = div64_u64_rem(y, tile->height, &y) * tile->stride * tile->height;
167
168 if (tile->tiling == I915_TILING_X) {
169 v += y * tile->width;
170 v += div64_u64_rem(x, tile->width, &x) << tile->size;
171 v += x;
172 } else if (tile->width == 128) {
173 const unsigned int ytile_span = 16;
174 const unsigned int ytile_height = 512;
175
176 v += y * ytile_span;
177 v += div64_u64_rem(x, ytile_span, &x) * ytile_height;
178 v += x;
179 } else {
180 const unsigned int ytile_span = 32;
181 const unsigned int ytile_height = 256;
182
183 v += y * ytile_span;
184 v += div64_u64_rem(x, ytile_span, &x) * ytile_height;
185 v += x;
186 }
187
188 switch (tile->swizzle) {
189 case I915_BIT_6_SWIZZLE_9:
190 v ^= swizzle_bit(9, v);
191 break;
192 case I915_BIT_6_SWIZZLE_9_10:
193 v ^= swizzle_bit(9, v) ^ swizzle_bit(10, v);
194 break;
195 case I915_BIT_6_SWIZZLE_9_11:
196 v ^= swizzle_bit(9, v) ^ swizzle_bit(11, v);
197 break;
198 case I915_BIT_6_SWIZZLE_9_10_11:
199 v ^= swizzle_bit(9, v) ^ swizzle_bit(10, v) ^ swizzle_bit(11, v);
200 break;
201 }
202
203 return v;
204 }
205
check_partial_mapping(struct drm_i915_gem_object * obj,const struct tile * tile,unsigned long end_time)206 static int check_partial_mapping(struct drm_i915_gem_object *obj,
207 const struct tile *tile,
208 unsigned long end_time)
209 {
210 const unsigned int nreal = obj->scratch / PAGE_SIZE;
211 const unsigned long npages = obj->base.size / PAGE_SIZE;
212 struct i915_vma *vma;
213 unsigned long page;
214 int err;
215
216 if (igt_timeout(end_time,
217 "%s: timed out before tiling=%d stride=%d\n",
218 __func__, tile->tiling, tile->stride))
219 return -EINTR;
220
221 err = i915_gem_object_set_tiling(obj, tile->tiling, tile->stride);
222 if (err) {
223 pr_err("Failed to set tiling mode=%u, stride=%u, err=%d\n",
224 tile->tiling, tile->stride, err);
225 return err;
226 }
227
228 GEM_BUG_ON(i915_gem_object_get_tiling(obj) != tile->tiling);
229 GEM_BUG_ON(i915_gem_object_get_stride(obj) != tile->stride);
230
231 for_each_prime_number_from(page, 1, npages) {
232 struct i915_ggtt_view view =
233 compute_partial_view(obj, page, MIN_CHUNK_PAGES);
234 u32 __iomem *io;
235 struct page *p;
236 unsigned int n;
237 u64 offset;
238 u32 *cpu;
239
240 GEM_BUG_ON(view.partial.size > nreal);
241
242 err = i915_gem_object_set_to_gtt_domain(obj, true);
243 if (err) {
244 pr_err("Failed to flush to GTT write domain; err=%d\n",
245 err);
246 return err;
247 }
248
249 vma = i915_gem_object_ggtt_pin(obj, &view, 0, 0, PIN_MAPPABLE);
250 if (IS_ERR(vma)) {
251 pr_err("Failed to pin partial view: offset=%lu; err=%d\n",
252 page, (int)PTR_ERR(vma));
253 return PTR_ERR(vma);
254 }
255
256 n = page - view.partial.offset;
257 GEM_BUG_ON(n >= view.partial.size);
258
259 io = i915_vma_pin_iomap(vma);
260 i915_vma_unpin(vma);
261 if (IS_ERR(io)) {
262 pr_err("Failed to iomap partial view: offset=%lu; err=%d\n",
263 page, (int)PTR_ERR(io));
264 return PTR_ERR(io);
265 }
266
267 iowrite32(page, io + n * PAGE_SIZE/sizeof(*io));
268 i915_vma_unpin_iomap(vma);
269
270 offset = tiled_offset(tile, page << PAGE_SHIFT);
271 if (offset >= obj->base.size)
272 continue;
273
274 flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU);
275
276 p = i915_gem_object_get_page(obj, offset >> PAGE_SHIFT);
277 cpu = kmap(p) + offset_in_page(offset);
278 drm_clflush_virt_range(cpu, sizeof(*cpu));
279 if (*cpu != (u32)page) {
280 pr_err("Partial view for %lu [%u] (offset=%llu, size=%u [%llu, row size %u], fence=%d, tiling=%d, stride=%d) misalignment, expected write to page (%llu + %u [0x%llx]) of 0x%x, found 0x%x\n",
281 page, n,
282 view.partial.offset,
283 view.partial.size,
284 vma->size >> PAGE_SHIFT,
285 tile_row_pages(obj),
286 vma->fence ? vma->fence->id : -1, tile->tiling, tile->stride,
287 offset >> PAGE_SHIFT,
288 (unsigned int)offset_in_page(offset),
289 offset,
290 (u32)page, *cpu);
291 err = -EINVAL;
292 }
293 *cpu = 0;
294 drm_clflush_virt_range(cpu, sizeof(*cpu));
295 kunmap(p);
296 if (err)
297 return err;
298
299 i915_vma_destroy(vma);
300 }
301
302 return 0;
303 }
304
igt_partial_tiling(void * arg)305 static int igt_partial_tiling(void *arg)
306 {
307 const unsigned int nreal = 1 << 12; /* largest tile row x2 */
308 struct drm_i915_private *i915 = arg;
309 struct drm_i915_gem_object *obj;
310 int tiling;
311 int err;
312
313 /* We want to check the page mapping and fencing of a large object
314 * mmapped through the GTT. The object we create is larger than can
315 * possibly be mmaped as a whole, and so we must use partial GGTT vma.
316 * We then check that a write through each partial GGTT vma ends up
317 * in the right set of pages within the object, and with the expected
318 * tiling, which we verify by manual swizzling.
319 */
320
321 obj = huge_gem_object(i915,
322 nreal << PAGE_SHIFT,
323 (1 + next_prime_number(i915->ggtt.vm.total >> PAGE_SHIFT)) << PAGE_SHIFT);
324 if (IS_ERR(obj))
325 return PTR_ERR(obj);
326
327 err = i915_gem_object_pin_pages(obj);
328 if (err) {
329 pr_err("Failed to allocate %u pages (%lu total), err=%d\n",
330 nreal, obj->base.size / PAGE_SIZE, err);
331 goto out;
332 }
333
334 mutex_lock(&i915->drm.struct_mutex);
335 intel_runtime_pm_get(i915);
336
337 if (1) {
338 IGT_TIMEOUT(end);
339 struct tile tile;
340
341 tile.height = 1;
342 tile.width = 1;
343 tile.size = 0;
344 tile.stride = 0;
345 tile.swizzle = I915_BIT_6_SWIZZLE_NONE;
346 tile.tiling = I915_TILING_NONE;
347
348 err = check_partial_mapping(obj, &tile, end);
349 if (err && err != -EINTR)
350 goto out_unlock;
351 }
352
353 for (tiling = I915_TILING_X; tiling <= I915_TILING_Y; tiling++) {
354 IGT_TIMEOUT(end);
355 unsigned int max_pitch;
356 unsigned int pitch;
357 struct tile tile;
358
359 if (i915->quirks & QUIRK_PIN_SWIZZLED_PAGES)
360 /*
361 * The swizzling pattern is actually unknown as it
362 * varies based on physical address of each page.
363 * See i915_gem_detect_bit_6_swizzle().
364 */
365 break;
366
367 tile.tiling = tiling;
368 switch (tiling) {
369 case I915_TILING_X:
370 tile.swizzle = i915->mm.bit_6_swizzle_x;
371 break;
372 case I915_TILING_Y:
373 tile.swizzle = i915->mm.bit_6_swizzle_y;
374 break;
375 }
376
377 GEM_BUG_ON(tile.swizzle == I915_BIT_6_SWIZZLE_UNKNOWN);
378 if (tile.swizzle == I915_BIT_6_SWIZZLE_9_17 ||
379 tile.swizzle == I915_BIT_6_SWIZZLE_9_10_17)
380 continue;
381
382 if (INTEL_GEN(i915) <= 2) {
383 tile.height = 16;
384 tile.width = 128;
385 tile.size = 11;
386 } else if (tile.tiling == I915_TILING_Y &&
387 HAS_128_BYTE_Y_TILING(i915)) {
388 tile.height = 32;
389 tile.width = 128;
390 tile.size = 12;
391 } else {
392 tile.height = 8;
393 tile.width = 512;
394 tile.size = 12;
395 }
396
397 if (INTEL_GEN(i915) < 4)
398 max_pitch = 8192 / tile.width;
399 else if (INTEL_GEN(i915) < 7)
400 max_pitch = 128 * I965_FENCE_MAX_PITCH_VAL / tile.width;
401 else
402 max_pitch = 128 * GEN7_FENCE_MAX_PITCH_VAL / tile.width;
403
404 for (pitch = max_pitch; pitch; pitch >>= 1) {
405 tile.stride = tile.width * pitch;
406 err = check_partial_mapping(obj, &tile, end);
407 if (err == -EINTR)
408 goto next_tiling;
409 if (err)
410 goto out_unlock;
411
412 if (pitch > 2 && INTEL_GEN(i915) >= 4) {
413 tile.stride = tile.width * (pitch - 1);
414 err = check_partial_mapping(obj, &tile, end);
415 if (err == -EINTR)
416 goto next_tiling;
417 if (err)
418 goto out_unlock;
419 }
420
421 if (pitch < max_pitch && INTEL_GEN(i915) >= 4) {
422 tile.stride = tile.width * (pitch + 1);
423 err = check_partial_mapping(obj, &tile, end);
424 if (err == -EINTR)
425 goto next_tiling;
426 if (err)
427 goto out_unlock;
428 }
429 }
430
431 if (INTEL_GEN(i915) >= 4) {
432 for_each_prime_number(pitch, max_pitch) {
433 tile.stride = tile.width * pitch;
434 err = check_partial_mapping(obj, &tile, end);
435 if (err == -EINTR)
436 goto next_tiling;
437 if (err)
438 goto out_unlock;
439 }
440 }
441
442 next_tiling: ;
443 }
444
445 out_unlock:
446 intel_runtime_pm_put(i915);
447 mutex_unlock(&i915->drm.struct_mutex);
448 i915_gem_object_unpin_pages(obj);
449 out:
450 i915_gem_object_put(obj);
451 return err;
452 }
453
make_obj_busy(struct drm_i915_gem_object * obj)454 static int make_obj_busy(struct drm_i915_gem_object *obj)
455 {
456 struct drm_i915_private *i915 = to_i915(obj->base.dev);
457 struct i915_request *rq;
458 struct i915_vma *vma;
459 int err;
460
461 vma = i915_vma_instance(obj, &i915->ggtt.vm, NULL);
462 if (IS_ERR(vma))
463 return PTR_ERR(vma);
464
465 err = i915_vma_pin(vma, 0, 0, PIN_USER);
466 if (err)
467 return err;
468
469 rq = i915_request_alloc(i915->engine[RCS], i915->kernel_context);
470 if (IS_ERR(rq)) {
471 i915_vma_unpin(vma);
472 return PTR_ERR(rq);
473 }
474
475 err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
476
477 i915_request_add(rq);
478
479 __i915_gem_object_release_unless_active(obj);
480 i915_vma_unpin(vma);
481
482 return err;
483 }
484
assert_mmap_offset(struct drm_i915_private * i915,unsigned long size,int expected)485 static bool assert_mmap_offset(struct drm_i915_private *i915,
486 unsigned long size,
487 int expected)
488 {
489 struct drm_i915_gem_object *obj;
490 int err;
491
492 obj = i915_gem_object_create_internal(i915, size);
493 if (IS_ERR(obj))
494 return PTR_ERR(obj);
495
496 err = i915_gem_object_create_mmap_offset(obj);
497 i915_gem_object_put(obj);
498
499 return err == expected;
500 }
501
disable_retire_worker(struct drm_i915_private * i915)502 static void disable_retire_worker(struct drm_i915_private *i915)
503 {
504 mutex_lock(&i915->drm.struct_mutex);
505 if (!i915->gt.active_requests++) {
506 intel_runtime_pm_get(i915);
507 i915_gem_unpark(i915);
508 intel_runtime_pm_put(i915);
509 }
510 mutex_unlock(&i915->drm.struct_mutex);
511 cancel_delayed_work_sync(&i915->gt.retire_work);
512 cancel_delayed_work_sync(&i915->gt.idle_work);
513 }
514
igt_mmap_offset_exhaustion(void * arg)515 static int igt_mmap_offset_exhaustion(void *arg)
516 {
517 struct drm_i915_private *i915 = arg;
518 struct drm_mm *mm = &i915->drm.vma_offset_manager->vm_addr_space_mm;
519 struct drm_i915_gem_object *obj;
520 struct drm_mm_node resv, *hole;
521 u64 hole_start, hole_end;
522 int loop, err;
523
524 /* Disable background reaper */
525 disable_retire_worker(i915);
526 GEM_BUG_ON(!i915->gt.awake);
527
528 /* Trim the device mmap space to only a page */
529 memset(&resv, 0, sizeof(resv));
530 drm_mm_for_each_hole(hole, mm, hole_start, hole_end) {
531 resv.start = hole_start;
532 resv.size = hole_end - hole_start - 1; /* PAGE_SIZE units */
533 err = drm_mm_reserve_node(mm, &resv);
534 if (err) {
535 pr_err("Failed to trim VMA manager, err=%d\n", err);
536 goto out_park;
537 }
538 break;
539 }
540
541 /* Just fits! */
542 if (!assert_mmap_offset(i915, PAGE_SIZE, 0)) {
543 pr_err("Unable to insert object into single page hole\n");
544 err = -EINVAL;
545 goto out;
546 }
547
548 /* Too large */
549 if (!assert_mmap_offset(i915, 2*PAGE_SIZE, -ENOSPC)) {
550 pr_err("Unexpectedly succeeded in inserting too large object into single page hole\n");
551 err = -EINVAL;
552 goto out;
553 }
554
555 /* Fill the hole, further allocation attempts should then fail */
556 obj = i915_gem_object_create_internal(i915, PAGE_SIZE);
557 if (IS_ERR(obj)) {
558 err = PTR_ERR(obj);
559 goto out;
560 }
561
562 err = i915_gem_object_create_mmap_offset(obj);
563 if (err) {
564 pr_err("Unable to insert object into reclaimed hole\n");
565 goto err_obj;
566 }
567
568 if (!assert_mmap_offset(i915, PAGE_SIZE, -ENOSPC)) {
569 pr_err("Unexpectedly succeeded in inserting object into no holes!\n");
570 err = -EINVAL;
571 goto err_obj;
572 }
573
574 i915_gem_object_put(obj);
575
576 /* Now fill with busy dead objects that we expect to reap */
577 for (loop = 0; loop < 3; loop++) {
578 if (i915_terminally_wedged(&i915->gpu_error))
579 break;
580
581 obj = i915_gem_object_create_internal(i915, PAGE_SIZE);
582 if (IS_ERR(obj)) {
583 err = PTR_ERR(obj);
584 goto out;
585 }
586
587 mutex_lock(&i915->drm.struct_mutex);
588 intel_runtime_pm_get(i915);
589 err = make_obj_busy(obj);
590 intel_runtime_pm_put(i915);
591 mutex_unlock(&i915->drm.struct_mutex);
592 if (err) {
593 pr_err("[loop %d] Failed to busy the object\n", loop);
594 goto err_obj;
595 }
596
597 /* NB we rely on the _active_ reference to access obj now */
598 GEM_BUG_ON(!i915_gem_object_is_active(obj));
599 err = i915_gem_object_create_mmap_offset(obj);
600 if (err) {
601 pr_err("[loop %d] i915_gem_object_create_mmap_offset failed with err=%d\n",
602 loop, err);
603 goto out;
604 }
605 }
606
607 out:
608 drm_mm_remove_node(&resv);
609 out_park:
610 mutex_lock(&i915->drm.struct_mutex);
611 if (--i915->gt.active_requests)
612 queue_delayed_work(i915->wq, &i915->gt.retire_work, 0);
613 else
614 queue_delayed_work(i915->wq, &i915->gt.idle_work, 0);
615 mutex_unlock(&i915->drm.struct_mutex);
616 return err;
617 err_obj:
618 i915_gem_object_put(obj);
619 goto out;
620 }
621
i915_gem_object_mock_selftests(void)622 int i915_gem_object_mock_selftests(void)
623 {
624 static const struct i915_subtest tests[] = {
625 SUBTEST(igt_gem_object),
626 SUBTEST(igt_phys_object),
627 };
628 struct drm_i915_private *i915;
629 int err;
630
631 i915 = mock_gem_device();
632 if (!i915)
633 return -ENOMEM;
634
635 err = i915_subtests(tests, i915);
636
637 drm_dev_put(&i915->drm);
638 return err;
639 }
640
i915_gem_object_live_selftests(struct drm_i915_private * i915)641 int i915_gem_object_live_selftests(struct drm_i915_private *i915)
642 {
643 static const struct i915_subtest tests[] = {
644 SUBTEST(igt_gem_huge),
645 SUBTEST(igt_partial_tiling),
646 SUBTEST(igt_mmap_offset_exhaustion),
647 };
648
649 return i915_subtests(tests, i915);
650 }
651