• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Copyright © 2011 Marek Olšák <maraeo@gmail.com>
3  * Copyright © 2015 Advanced Micro Devices, Inc.
4  * All Rights Reserved.
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining
7  * a copy of this software and associated documentation files (the
8  * "Software"), to deal in the Software without restriction, including
9  * without limitation the rights to use, copy, modify, merge, publish,
10  * distribute, sub license, and/or sell copies of the Software, and to
11  * permit persons to whom the Software is furnished to do so, subject to
12  * the following conditions:
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
15  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
16  * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
17  * NON-INFRINGEMENT. IN NO EVENT SHALL THE COPYRIGHT HOLDERS, AUTHORS
18  * AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
21  * USE OR OTHER DEALINGS IN THE SOFTWARE.
22  *
23  * The above copyright notice and this permission notice (including the
24  * next paragraph) shall be included in all copies or substantial portions
25  * of the Software.
26  */
27 
28 #ifndef AMDGPU_CS_H
29 #define AMDGPU_CS_H
30 
31 #include "amdgpu_bo.h"
32 #include "util/u_memory.h"
33 #include <amdgpu_drm.h>
34 
35 struct amdgpu_ctx {
36    struct amdgpu_winsys *ws;
37    amdgpu_context_handle ctx;
38    amdgpu_bo_handle user_fence_bo;
39    uint64_t *user_fence_cpu_address_base;
40    int refcount;
41    unsigned initial_num_total_rejected_cs;
42    unsigned num_rejected_cs;
43 };
44 
45 struct amdgpu_cs_buffer {
46    struct amdgpu_winsys_bo *bo;
47    union {
48       struct {
49          uint64_t priority_usage;
50       } real;
51       struct {
52          uint32_t real_idx; /* index of underlying real BO */
53       } slab;
54    } u;
55    enum radeon_bo_usage usage;
56 };
57 
58 enum ib_type {
59    IB_MAIN,
60    IB_NUM,
61 };
62 
63 struct amdgpu_ib {
64    struct radeon_winsys_cs base;
65 
66    /* A buffer out of which new IBs are allocated. */
67    struct pb_buffer        *big_ib_buffer;
68    uint8_t                 *ib_mapped;
69    unsigned                used_ib_space;
70    unsigned                max_ib_size;
71    uint32_t                *ptr_ib_size;
72    bool                    ptr_ib_size_inside_ib;
73    enum ib_type            ib_type;
74 };
75 
76 struct amdgpu_cs_context {
77    struct drm_amdgpu_cs_chunk_ib ib[IB_NUM];
78 
79    /* Buffers. */
80    unsigned                    max_real_buffers;
81    unsigned                    num_real_buffers;
82    struct amdgpu_cs_buffer     *real_buffers;
83 
84    unsigned                    max_real_submit;
85    amdgpu_bo_handle            *handles;
86    uint8_t                     *flags;
87 
88    unsigned                    num_slab_buffers;
89    unsigned                    max_slab_buffers;
90    struct amdgpu_cs_buffer     *slab_buffers;
91 
92    unsigned                    num_sparse_buffers;
93    unsigned                    max_sparse_buffers;
94    struct amdgpu_cs_buffer     *sparse_buffers;
95 
96    int                         buffer_indices_hashlist[4096];
97 
98    struct amdgpu_winsys_bo     *last_added_bo;
99    unsigned                    last_added_bo_index;
100    unsigned                    last_added_bo_usage;
101    uint64_t                    last_added_bo_priority_usage;
102 
103    struct pipe_fence_handle    **fence_dependencies;
104    unsigned                    num_fence_dependencies;
105    unsigned                    max_fence_dependencies;
106 
107    struct pipe_fence_handle    *fence;
108 
109    /* the error returned from cs_flush for non-async submissions */
110    int                         error_code;
111 };
112 
113 struct amdgpu_cs {
114    struct amdgpu_ib main; /* must be first because this is inherited */
115    struct amdgpu_ctx *ctx;
116    enum ring_type ring_type;
117    struct drm_amdgpu_cs_chunk_fence fence_chunk;
118 
119    /* We flip between these two CS. While one is being consumed
120     * by the kernel in another thread, the other one is being filled
121     * by the pipe driver. */
122    struct amdgpu_cs_context csc1;
123    struct amdgpu_cs_context csc2;
124    /* The currently-used CS. */
125    struct amdgpu_cs_context *csc;
126    /* The CS being currently-owned by the other thread. */
127    struct amdgpu_cs_context *cst;
128 
129    /* Flush CS. */
130    void (*flush_cs)(void *ctx, unsigned flags, struct pipe_fence_handle **fence);
131    void *flush_data;
132 
133    struct util_queue_fence flush_completed;
134    struct pipe_fence_handle *next_fence;
135 };
136 
137 struct amdgpu_fence {
138    struct pipe_reference reference;
139    /* If ctx == NULL, this fence is syncobj-based. */
140    uint32_t syncobj;
141 
142    struct amdgpu_winsys *ws;
143    struct amdgpu_ctx *ctx;  /* submission context */
144    struct amdgpu_cs_fence fence;
145    uint64_t *user_fence_cpu_address;
146 
147    /* If the fence has been submitted. This is unsignalled for deferred fences
148     * (cs->next_fence) and while an IB is still being submitted in the submit
149     * thread. */
150    struct util_queue_fence submitted;
151 
152    volatile int signalled;              /* bool (int for atomicity) */
153 };
154 
amdgpu_fence_is_syncobj(struct amdgpu_fence * fence)155 static inline bool amdgpu_fence_is_syncobj(struct amdgpu_fence *fence)
156 {
157    return fence->ctx == NULL;
158 }
159 
amdgpu_ctx_unref(struct amdgpu_ctx * ctx)160 static inline void amdgpu_ctx_unref(struct amdgpu_ctx *ctx)
161 {
162    if (p_atomic_dec_zero(&ctx->refcount)) {
163       amdgpu_cs_ctx_free(ctx->ctx);
164       amdgpu_bo_free(ctx->user_fence_bo);
165       FREE(ctx);
166    }
167 }
168 
amdgpu_fence_reference(struct pipe_fence_handle ** dst,struct pipe_fence_handle * src)169 static inline void amdgpu_fence_reference(struct pipe_fence_handle **dst,
170                                           struct pipe_fence_handle *src)
171 {
172    struct amdgpu_fence **rdst = (struct amdgpu_fence **)dst;
173    struct amdgpu_fence *rsrc = (struct amdgpu_fence *)src;
174 
175    if (pipe_reference(&(*rdst)->reference, &rsrc->reference)) {
176       struct amdgpu_fence *fence = *rdst;
177 
178       if (amdgpu_fence_is_syncobj(fence))
179          amdgpu_cs_destroy_syncobj(fence->ws->dev, fence->syncobj);
180       else
181          amdgpu_ctx_unref(fence->ctx);
182 
183       util_queue_fence_destroy(&fence->submitted);
184       FREE(fence);
185    }
186    *rdst = rsrc;
187 }
188 
189 int amdgpu_lookup_buffer(struct amdgpu_cs_context *cs, struct amdgpu_winsys_bo *bo);
190 
191 static inline struct amdgpu_ib *
amdgpu_ib(struct radeon_winsys_cs * base)192 amdgpu_ib(struct radeon_winsys_cs *base)
193 {
194    return (struct amdgpu_ib *)base;
195 }
196 
197 static inline struct amdgpu_cs *
amdgpu_cs(struct radeon_winsys_cs * base)198 amdgpu_cs(struct radeon_winsys_cs *base)
199 {
200    assert(amdgpu_ib(base)->ib_type == IB_MAIN);
201    return (struct amdgpu_cs*)base;
202 }
203 
204 #define get_container(member_ptr, container_type, container_member) \
205    (container_type *)((char *)(member_ptr) - offsetof(container_type, container_member))
206 
207 static inline struct amdgpu_cs *
amdgpu_cs_from_ib(struct amdgpu_ib * ib)208 amdgpu_cs_from_ib(struct amdgpu_ib *ib)
209 {
210    switch (ib->ib_type) {
211    case IB_MAIN:
212       return get_container(ib, struct amdgpu_cs, main);
213    default:
214       unreachable("bad ib_type");
215    }
216 }
217 
218 static inline bool
amdgpu_bo_is_referenced_by_cs(struct amdgpu_cs * cs,struct amdgpu_winsys_bo * bo)219 amdgpu_bo_is_referenced_by_cs(struct amdgpu_cs *cs,
220                               struct amdgpu_winsys_bo *bo)
221 {
222    int num_refs = bo->num_cs_references;
223    return num_refs == bo->ws->num_cs ||
224          (num_refs && amdgpu_lookup_buffer(cs->csc, bo) != -1);
225 }
226 
227 static inline bool
amdgpu_bo_is_referenced_by_cs_with_usage(struct amdgpu_cs * cs,struct amdgpu_winsys_bo * bo,enum radeon_bo_usage usage)228 amdgpu_bo_is_referenced_by_cs_with_usage(struct amdgpu_cs *cs,
229                                          struct amdgpu_winsys_bo *bo,
230                                          enum radeon_bo_usage usage)
231 {
232    int index;
233    struct amdgpu_cs_buffer *buffer;
234 
235    if (!bo->num_cs_references)
236       return false;
237 
238    index = amdgpu_lookup_buffer(cs->csc, bo);
239    if (index == -1)
240       return false;
241 
242    buffer = bo->bo ? &cs->csc->real_buffers[index] :
243             bo->sparse ? &cs->csc->sparse_buffers[index] :
244             &cs->csc->slab_buffers[index];
245 
246    return (buffer->usage & usage) != 0;
247 }
248 
249 static inline bool
amdgpu_bo_is_referenced_by_any_cs(struct amdgpu_winsys_bo * bo)250 amdgpu_bo_is_referenced_by_any_cs(struct amdgpu_winsys_bo *bo)
251 {
252    return bo->num_cs_references != 0;
253 }
254 
255 bool amdgpu_fence_wait(struct pipe_fence_handle *fence, uint64_t timeout,
256                        bool absolute);
257 void amdgpu_add_fences(struct amdgpu_winsys_bo *bo,
258                        unsigned num_fences,
259                        struct pipe_fence_handle **fences);
260 void amdgpu_cs_sync_flush(struct radeon_winsys_cs *rcs);
261 void amdgpu_cs_init_functions(struct amdgpu_winsys *ws);
262 void amdgpu_cs_submit_ib(void *job, int thread_index);
263 
264 #endif
265