1 /*
2 * Copyright © 2017 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included
12 * in all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
15 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23 #include <stdio.h>
24 #include <time.h>
25 #include "pipe/p_defines.h"
26 #include "pipe/p_state.h"
27 #include "util/u_debug.h"
28 #include "util/ralloc.h"
29 #include "util/u_inlines.h"
30 #include "util/format/u_format.h"
31 #include "util/u_upload_mgr.h"
32 #include "iris_context.h"
33 #include "iris_resource.h"
34 #include "iris_screen.h"
35 #include "iris_utrace.h"
36 #include "common/intel_sample_positions.h"
37
38 /**
39 * The pipe->set_debug_callback() driver hook.
40 */
41 static void
iris_set_debug_callback(struct pipe_context * ctx,const struct util_debug_callback * cb)42 iris_set_debug_callback(struct pipe_context *ctx,
43 const struct util_debug_callback *cb)
44 {
45 struct iris_context *ice = (struct iris_context *)ctx;
46 struct iris_screen *screen = (struct iris_screen *)ctx->screen;
47
48 util_queue_finish(&screen->shader_compiler_queue);
49
50 if (cb)
51 ice->dbg = *cb;
52 else
53 memset(&ice->dbg, 0, sizeof(ice->dbg));
54 }
55
56 /**
57 * Called from the batch module when it detects a GPU hang.
58 *
59 * In this case, we've lost our GEM context, and can't rely on any existing
60 * state on the GPU. We must mark everything dirty and wipe away any saved
61 * assumptions about the last known state of the GPU.
62 */
63 void
iris_lost_context_state(struct iris_batch * batch)64 iris_lost_context_state(struct iris_batch *batch)
65 {
66 struct iris_context *ice = batch->ice;
67
68 if (batch->name == IRIS_BATCH_RENDER) {
69 batch->screen->vtbl.init_render_context(batch);
70 } else if (batch->name == IRIS_BATCH_COMPUTE) {
71 batch->screen->vtbl.init_compute_context(batch);
72 } else if (batch->name == IRIS_BATCH_BLITTER) {
73 /* No state to set up */
74 } else {
75 unreachable("unhandled batch reset");
76 }
77
78 ice->state.dirty = ~0ull;
79 ice->state.stage_dirty = ~0ull;
80 ice->state.current_hash_scale = 0;
81 memset(&ice->shaders.urb, 0, sizeof(ice->shaders.urb));
82 memset(ice->state.last_block, 0, sizeof(ice->state.last_block));
83 memset(ice->state.last_grid, 0, sizeof(ice->state.last_grid));
84 ice->state.last_grid_dim = 0;
85 batch->last_binder_address = ~0ull;
86 batch->last_aux_map_state = 0;
87 batch->screen->vtbl.lost_genx_state(ice, batch);
88 }
89
90 static enum pipe_reset_status
iris_get_device_reset_status(struct pipe_context * ctx)91 iris_get_device_reset_status(struct pipe_context *ctx)
92 {
93 struct iris_context *ice = (struct iris_context *)ctx;
94
95 enum pipe_reset_status worst_reset = PIPE_NO_RESET;
96
97 /* Check the reset status of each batch's hardware context, and take the
98 * worst status (if one was guilty, proclaim guilt).
99 */
100 iris_foreach_batch(ice, batch) {
101 enum pipe_reset_status batch_reset =
102 iris_batch_check_for_reset(batch);
103
104 if (batch_reset == PIPE_NO_RESET)
105 continue;
106
107 if (worst_reset == PIPE_NO_RESET) {
108 worst_reset = batch_reset;
109 } else {
110 /* GUILTY < INNOCENT < UNKNOWN */
111 worst_reset = MIN2(worst_reset, batch_reset);
112 }
113 }
114
115 if (worst_reset != PIPE_NO_RESET && ice->reset.reset)
116 ice->reset.reset(ice->reset.data, worst_reset);
117
118 return worst_reset;
119 }
120
121 static void
iris_set_device_reset_callback(struct pipe_context * ctx,const struct pipe_device_reset_callback * cb)122 iris_set_device_reset_callback(struct pipe_context *ctx,
123 const struct pipe_device_reset_callback *cb)
124 {
125 struct iris_context *ice = (struct iris_context *)ctx;
126
127 if (cb)
128 ice->reset = *cb;
129 else
130 memset(&ice->reset, 0, sizeof(ice->reset));
131 }
132
133 static void
iris_get_sample_position(struct pipe_context * ctx,unsigned sample_count,unsigned sample_index,float * out_value)134 iris_get_sample_position(struct pipe_context *ctx,
135 unsigned sample_count,
136 unsigned sample_index,
137 float *out_value)
138 {
139 union {
140 struct {
141 float x[16];
142 float y[16];
143 } a;
144 struct {
145 float _0XOffset, _1XOffset, _2XOffset, _3XOffset,
146 _4XOffset, _5XOffset, _6XOffset, _7XOffset,
147 _8XOffset, _9XOffset, _10XOffset, _11XOffset,
148 _12XOffset, _13XOffset, _14XOffset, _15XOffset;
149 float _0YOffset, _1YOffset, _2YOffset, _3YOffset,
150 _4YOffset, _5YOffset, _6YOffset, _7YOffset,
151 _8YOffset, _9YOffset, _10YOffset, _11YOffset,
152 _12YOffset, _13YOffset, _14YOffset, _15YOffset;
153 } v;
154 } u;
155 switch (sample_count) {
156 case 1: INTEL_SAMPLE_POS_1X(u.v._); break;
157 case 2: INTEL_SAMPLE_POS_2X(u.v._); break;
158 case 4: INTEL_SAMPLE_POS_4X(u.v._); break;
159 case 8: INTEL_SAMPLE_POS_8X(u.v._); break;
160 case 16: INTEL_SAMPLE_POS_16X(u.v._); break;
161 default: unreachable("invalid sample count");
162 }
163
164 out_value[0] = u.a.x[sample_index];
165 out_value[1] = u.a.y[sample_index];
166 }
167
168 static bool
create_dirty_dmabuf_set(struct iris_context * ice)169 create_dirty_dmabuf_set(struct iris_context *ice)
170 {
171 assert(ice->dirty_dmabufs == NULL);
172
173 ice->dirty_dmabufs = _mesa_pointer_set_create(ice);
174 return ice->dirty_dmabufs != NULL;
175 }
176
177 void
iris_mark_dirty_dmabuf(struct iris_context * ice,struct pipe_resource * res)178 iris_mark_dirty_dmabuf(struct iris_context *ice,
179 struct pipe_resource *res)
180 {
181 if (!_mesa_set_search(ice->dirty_dmabufs, res)) {
182 _mesa_set_add(ice->dirty_dmabufs, res);
183 pipe_reference(NULL, &res->reference);
184 }
185 }
186
187 static void
clear_dirty_dmabuf_set(struct iris_context * ice)188 clear_dirty_dmabuf_set(struct iris_context *ice)
189 {
190 set_foreach(ice->dirty_dmabufs, entry) {
191 struct pipe_resource *res = (struct pipe_resource *)entry->key;
192 if (pipe_reference(&res->reference, NULL))
193 res->screen->resource_destroy(res->screen, res);
194 }
195
196 _mesa_set_clear(ice->dirty_dmabufs, NULL);
197 }
198
199 void
iris_flush_dirty_dmabufs(struct iris_context * ice)200 iris_flush_dirty_dmabufs(struct iris_context *ice)
201 {
202 set_foreach(ice->dirty_dmabufs, entry) {
203 struct pipe_resource *res = (struct pipe_resource *)entry->key;
204 ice->ctx.flush_resource(&ice->ctx, res);
205 }
206
207 clear_dirty_dmabuf_set(ice);
208 }
209
210 /**
211 * Destroy a context, freeing any associated memory.
212 */
213 void
iris_destroy_context(struct pipe_context * ctx)214 iris_destroy_context(struct pipe_context *ctx)
215 {
216 struct iris_context *ice = (struct iris_context *)ctx;
217 struct iris_screen *screen = (struct iris_screen *)ctx->screen;
218
219 blorp_finish(&ice->blorp);
220
221 if (ctx->stream_uploader)
222 u_upload_destroy(ctx->stream_uploader);
223 if (ctx->const_uploader)
224 u_upload_destroy(ctx->const_uploader);
225
226 clear_dirty_dmabuf_set(ice);
227
228 screen->vtbl.destroy_state(ice);
229
230 for (unsigned i = 0; i < ARRAY_SIZE(ice->shaders.scratch_surfs); i++)
231 pipe_resource_reference(&ice->shaders.scratch_surfs[i].res, NULL);
232
233 for (unsigned i = 0; i < ARRAY_SIZE(ice->shaders.scratch_bos); i++) {
234 for (unsigned j = 0; j < ARRAY_SIZE(ice->shaders.scratch_bos[i]); j++)
235 iris_bo_unreference(ice->shaders.scratch_bos[i][j]);
236 }
237
238 iris_destroy_program_cache(ice);
239 if (screen->measure.config)
240 iris_destroy_ctx_measure(ice);
241
242 u_upload_destroy(ice->state.surface_uploader);
243 u_upload_destroy(ice->state.scratch_surface_uploader);
244 u_upload_destroy(ice->state.dynamic_uploader);
245 u_upload_destroy(ice->query_buffer_uploader);
246
247 iris_destroy_batches(ice);
248 iris_destroy_binder(&ice->state.binder);
249
250 iris_utrace_fini(ice);
251
252 slab_destroy_child(&ice->transfer_pool);
253 slab_destroy_child(&ice->transfer_pool_unsync);
254
255 ralloc_free(ice);
256 }
257
258 #define genX_call(devinfo, func, ...) \
259 switch ((devinfo)->verx10) { \
260 case 200: \
261 gfx20_##func(__VA_ARGS__); \
262 break; \
263 case 125: \
264 gfx125_##func(__VA_ARGS__); \
265 break; \
266 case 120: \
267 gfx12_##func(__VA_ARGS__); \
268 break; \
269 case 110: \
270 gfx11_##func(__VA_ARGS__); \
271 break; \
272 case 90: \
273 gfx9_##func(__VA_ARGS__); \
274 break; \
275 case 80: \
276 gfx8_##func(__VA_ARGS__); \
277 break; \
278 default: \
279 unreachable("Unknown hardware generation"); \
280 }
281
282 /**
283 * Create a context.
284 *
285 * This is where each context begins.
286 */
287 struct pipe_context *
iris_create_context(struct pipe_screen * pscreen,void * priv,unsigned flags)288 iris_create_context(struct pipe_screen *pscreen, void *priv, unsigned flags)
289 {
290 struct iris_screen *screen = (struct iris_screen*)pscreen;
291 const struct intel_device_info *devinfo = screen->devinfo;
292 struct iris_context *ice = rzalloc(NULL, struct iris_context);
293
294 if (!ice)
295 return NULL;
296
297 struct pipe_context *ctx = &ice->ctx;
298
299 ctx->screen = pscreen;
300 ctx->priv = priv;
301
302 ctx->stream_uploader = u_upload_create_default(ctx);
303 if (!ctx->stream_uploader) {
304 ralloc_free(ice);
305 return NULL;
306 }
307 ctx->const_uploader = u_upload_create(ctx, 1024 * 1024,
308 PIPE_BIND_CONSTANT_BUFFER,
309 PIPE_USAGE_IMMUTABLE,
310 IRIS_RESOURCE_FLAG_DEVICE_MEM);
311 if (!ctx->const_uploader) {
312 u_upload_destroy(ctx->stream_uploader);
313 ralloc_free(ice);
314 return NULL;
315 }
316
317 if (!create_dirty_dmabuf_set(ice)) {
318 ralloc_free(ice);
319 return NULL;
320 }
321
322 ctx->destroy = iris_destroy_context;
323 ctx->set_debug_callback = iris_set_debug_callback;
324 ctx->set_device_reset_callback = iris_set_device_reset_callback;
325 ctx->get_device_reset_status = iris_get_device_reset_status;
326 ctx->get_sample_position = iris_get_sample_position;
327
328 iris_init_context_fence_functions(ctx);
329 iris_init_blit_functions(ctx);
330 iris_init_clear_functions(ctx);
331 iris_init_program_functions(ctx);
332 iris_init_resource_functions(ctx);
333 iris_init_flush_functions(ctx);
334 iris_init_perfquery_functions(ctx);
335
336 iris_init_program_cache(ice);
337 iris_init_binder(ice);
338
339 slab_create_child(&ice->transfer_pool, &screen->transfer_pool);
340 slab_create_child(&ice->transfer_pool_unsync, &screen->transfer_pool);
341
342 ice->state.surface_uploader =
343 u_upload_create(ctx, 64 * 1024, PIPE_BIND_CUSTOM, PIPE_USAGE_IMMUTABLE,
344 IRIS_RESOURCE_FLAG_SURFACE_MEMZONE |
345 IRIS_RESOURCE_FLAG_DEVICE_MEM);
346 ice->state.scratch_surface_uploader =
347 u_upload_create(ctx, 64 * 1024, PIPE_BIND_CUSTOM, PIPE_USAGE_IMMUTABLE,
348 IRIS_RESOURCE_FLAG_SCRATCH_MEMZONE |
349 IRIS_RESOURCE_FLAG_DEVICE_MEM);
350 ice->state.dynamic_uploader =
351 u_upload_create(ctx, 64 * 1024, PIPE_BIND_CUSTOM, PIPE_USAGE_IMMUTABLE,
352 IRIS_RESOURCE_FLAG_DYNAMIC_MEMZONE |
353 IRIS_RESOURCE_FLAG_DEVICE_MEM);
354
355 ice->query_buffer_uploader =
356 u_upload_create(ctx, 16 * 1024, PIPE_BIND_CUSTOM, PIPE_USAGE_STAGING,
357 0);
358
359 genX_call(devinfo, init_state, ice);
360 genX_call(devinfo, init_blorp, ice);
361 genX_call(devinfo, init_query, ice);
362
363 if (flags & PIPE_CONTEXT_HIGH_PRIORITY)
364 ice->priority = IRIS_CONTEXT_HIGH_PRIORITY;
365 if (flags & PIPE_CONTEXT_LOW_PRIORITY)
366 ice->priority = IRIS_CONTEXT_LOW_PRIORITY;
367 if (flags & PIPE_CONTEXT_PROTECTED)
368 ice->protected = true;
369
370 if (INTEL_DEBUG(DEBUG_BATCH))
371 ice->state.sizes = _mesa_hash_table_u64_create(ice);
372
373 /* Do this before initializing the batches */
374 iris_utrace_init(ice);
375
376 iris_init_batches(ice);
377
378 screen->vtbl.init_render_context(&ice->batches[IRIS_BATCH_RENDER]);
379 screen->vtbl.init_compute_context(&ice->batches[IRIS_BATCH_COMPUTE]);
380 screen->vtbl.init_copy_context(&ice->batches[IRIS_BATCH_BLITTER]);
381
382 if (!(flags & PIPE_CONTEXT_PREFER_THREADED))
383 return ctx;
384
385 /* Clover doesn't support u_threaded_context */
386 if (flags & PIPE_CONTEXT_COMPUTE_ONLY)
387 return ctx;
388
389 return threaded_context_create(ctx, &screen->transfer_pool,
390 iris_replace_buffer_storage,
391 &(struct threaded_context_options){
392 .unsynchronized_get_device_reset_status = true,
393 },
394 &ice->thrctx);
395 }
396