1 /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
2 /*
3 *
4 * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved.
5 *
6 * This program is free software and is provided to you under the terms of the
7 * GNU General Public License version 2 as published by the Free Software
8 * Foundation, and any use by you of this program is subject to the terms
9 * of such GNU license.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, you can access it online at
18 * http://www.gnu.org/licenses/gpl-2.0.html.
19 *
20 */
21
22 /*
23 * Definitions (types, defines, etcs) specific to Job Manager Kbase.
24 * They are placed here to allow the hierarchy of header files to work.
25 */
26
27 #ifndef _KBASE_JM_DEFS_H_
28 #define _KBASE_JM_DEFS_H_
29
30 #include "mali_kbase_js_defs.h"
31
32 /* Dump Job slot trace on error (only active if KBASE_KTRACE_ENABLE != 0) */
33 #define KBASE_KTRACE_DUMP_ON_JOB_SLOT_ERROR 1
34
35 /*
36 * Number of milliseconds before resetting the GPU when a job cannot be "zapped"
37 * from the hardware. Note that the time is actually
38 * ZAP_TIMEOUT+SOFT_STOP_RESET_TIMEOUT between the context zap starting and
39 * the GPU actually being reset to give other contexts time for their jobs
40 * to be soft-stopped and removed from the hardware before resetting.
41 */
42 #define ZAP_TIMEOUT 1000
43
44 /*
45 * Prevent soft-stops from occurring in scheduling situations
46 *
47 * This is not due to HW issues, but when scheduling is desired to be more
48 * predictable.
49 *
50 * Therefore, soft stop may still be disabled due to HW issues.
51 *
52 * Soft stop will still be used for non-scheduling purposes e.g. when
53 * terminating a context.
54 *
55 * if not in use, define this value to 0 instead of being undefined.
56 */
57 #define KBASE_DISABLE_SCHEDULING_SOFT_STOPS 0
58
59 /*
60 * Prevent hard-stops from occurring in scheduling situations
61 *
62 * This is not due to HW issues, but when scheduling is desired to be more
63 * predictable.
64 *
65 * Hard stop will still be used for non-scheduling purposes e.g. when
66 * terminating a context.
67 *
68 * if not in use, define this value to 0 instead of being undefined.
69 */
70 #define KBASE_DISABLE_SCHEDULING_HARD_STOPS 0
71
72 /* Atom has been previously soft-stopped */
73 #define KBASE_KATOM_FLAG_BEEN_SOFT_STOPPED (1<<1)
74 /* Atom has been previously retried to execute */
75 #define KBASE_KATOM_FLAGS_RERUN (1<<2)
76 /* Atom submitted with JOB_CHAIN_FLAG bit set in JS_CONFIG_NEXT register, helps
77 * to disambiguate short-running job chains during soft/hard stopping of jobs
78 */
79 #define KBASE_KATOM_FLAGS_JOBCHAIN (1<<3)
80 /* Atom has been previously hard-stopped. */
81 #define KBASE_KATOM_FLAG_BEEN_HARD_STOPPED (1<<4)
82 /* Atom has caused us to enter disjoint state */
83 #define KBASE_KATOM_FLAG_IN_DISJOINT (1<<5)
84 /* Atom blocked on cross-slot dependency */
85 #define KBASE_KATOM_FLAG_X_DEP_BLOCKED (1<<7)
86 /* Atom has fail dependency on cross-slot dependency */
87 #define KBASE_KATOM_FLAG_FAIL_BLOCKER (1<<8)
88 /* Atom is currently in the list of atoms blocked on cross-slot dependencies */
89 #define KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST (1<<9)
90 /* Atom requires GPU to be in protected mode */
91 #define KBASE_KATOM_FLAG_PROTECTED (1<<11)
92 /* Atom has been stored in runnable_tree */
93 #define KBASE_KATOM_FLAG_JSCTX_IN_TREE (1<<12)
94 /* Atom is waiting for L2 caches to power up in order to enter protected mode */
95 #define KBASE_KATOM_FLAG_HOLDING_L2_REF_PROT (1<<13)
96
97 /* SW related flags about types of JS_COMMAND action
98 * NOTE: These must be masked off by JS_COMMAND_MASK
99 */
100
101 /* This command causes a disjoint event */
102 #define JS_COMMAND_SW_CAUSES_DISJOINT 0x100
103
104 /* Bitmask of all SW related flags */
105 #define JS_COMMAND_SW_BITS (JS_COMMAND_SW_CAUSES_DISJOINT)
106
107 #if (JS_COMMAND_SW_BITS & JS_COMMAND_MASK)
108 #error "JS_COMMAND_SW_BITS not masked off by JS_COMMAND_MASK." \
109 "Must update JS_COMMAND_SW_<..> bitmasks"
110 #endif
111
112 /* Soft-stop command that causes a Disjoint event. This of course isn't
113 * entirely masked off by JS_COMMAND_MASK
114 */
115 #define JS_COMMAND_SOFT_STOP_WITH_SW_DISJOINT \
116 (JS_COMMAND_SW_CAUSES_DISJOINT | JS_COMMAND_SOFT_STOP)
117
118 #define KBASEP_ATOM_ID_INVALID BASE_JD_ATOM_COUNT
119
120 /* Serialize atoms within a slot (ie only one atom per job slot) */
121 #define KBASE_SERIALIZE_INTRA_SLOT (1 << 0)
122 /* Serialize atoms between slots (ie only one job slot running at any time) */
123 #define KBASE_SERIALIZE_INTER_SLOT (1 << 1)
124 /* Reset the GPU after each atom completion */
125 #define KBASE_SERIALIZE_RESET (1 << 2)
126
127 #if IS_ENABLED(CONFIG_DEBUG_FS)
128 /**
129 * struct base_job_fault_event - keeps track of the atom which faulted or which
130 * completed after the faulty atom but before the
131 * debug data for faulty atom was dumped.
132 *
133 * @event_code: event code for the atom, should != BASE_JD_EVENT_DONE for
134 * the atom which faulted.
135 * @katom: pointer to the atom for which job fault occurred or which
136 * completed after the faulty atom.
137 * @job_fault_work: work item, queued only for the faulty atom, which waits for
138 * the dumping to get completed and then does the bottom half
139 * of job done for the atoms which followed the faulty atom.
140 * @head: List head used to store the atom in the global list of
141 * faulty atoms or context specific list of atoms which got
142 * completed during the dump.
143 * @reg_offset: offset of the register to be dumped next, only applicable
144 * for the faulty atom.
145 */
146 struct base_job_fault_event {
147
148 u32 event_code;
149 struct kbase_jd_atom *katom;
150 struct work_struct job_fault_work;
151 struct list_head head;
152 int reg_offset;
153 };
154 #endif
155
156 /**
157 * struct kbase_jd_atom_dependency - Contains the dependency info for an atom.
158 * @atom: pointer to the dependee atom.
159 * @dep_type: type of dependency on the dependee @atom, i.e. order or data
160 * dependency. BASE_JD_DEP_TYPE_INVALID indicates no dependency.
161 */
162 struct kbase_jd_atom_dependency {
163 struct kbase_jd_atom *atom;
164 u8 dep_type;
165 };
166
167 /**
168 * kbase_jd_katom_dep_atom - Retrieves a read-only reference to the
169 * dependee atom.
170 * @dep: pointer to the dependency info structure.
171 *
172 * Return: readonly reference to dependee atom.
173 */
174 static inline const struct kbase_jd_atom *
kbase_jd_katom_dep_atom(const struct kbase_jd_atom_dependency * dep)175 kbase_jd_katom_dep_atom(const struct kbase_jd_atom_dependency *dep)
176 {
177 KBASE_DEBUG_ASSERT(dep != NULL);
178
179 return (const struct kbase_jd_atom *)(dep->atom);
180 }
181
182 /**
183 * kbase_jd_katom_dep_type - Retrieves the dependency type info
184 *
185 * @dep: pointer to the dependency info structure.
186 *
187 * Return: the type of dependency there is on the dependee atom.
188 */
kbase_jd_katom_dep_type(const struct kbase_jd_atom_dependency * dep)189 static inline u8 kbase_jd_katom_dep_type(
190 const struct kbase_jd_atom_dependency *dep)
191 {
192 KBASE_DEBUG_ASSERT(dep != NULL);
193
194 return dep->dep_type;
195 }
196
197 /**
198 * kbase_jd_katom_dep_set - sets up the dependency info structure
199 * as per the values passed.
200 * @const_dep: pointer to the dependency info structure to be setup.
201 * @a: pointer to the dependee atom.
202 * @type: type of dependency there is on the dependee atom.
203 */
kbase_jd_katom_dep_set(const struct kbase_jd_atom_dependency * const_dep,struct kbase_jd_atom * a,u8 type)204 static inline void kbase_jd_katom_dep_set(
205 const struct kbase_jd_atom_dependency *const_dep,
206 struct kbase_jd_atom *a, u8 type)
207 {
208 struct kbase_jd_atom_dependency *dep;
209
210 KBASE_DEBUG_ASSERT(const_dep != NULL);
211
212 dep = (struct kbase_jd_atom_dependency *)const_dep;
213
214 dep->atom = a;
215 dep->dep_type = type;
216 }
217
218 /**
219 * kbase_jd_katom_dep_clear - resets the dependency info structure
220 *
221 * @const_dep: pointer to the dependency info structure to be setup.
222 */
kbase_jd_katom_dep_clear(const struct kbase_jd_atom_dependency * const_dep)223 static inline void kbase_jd_katom_dep_clear(
224 const struct kbase_jd_atom_dependency *const_dep)
225 {
226 struct kbase_jd_atom_dependency *dep;
227
228 KBASE_DEBUG_ASSERT(const_dep != NULL);
229
230 dep = (struct kbase_jd_atom_dependency *)const_dep;
231
232 dep->atom = NULL;
233 dep->dep_type = BASE_JD_DEP_TYPE_INVALID;
234 }
235
236 /**
237 * enum kbase_atom_gpu_rb_state - The state of an atom, pertinent after it
238 * becomes runnable, with respect to job slot
239 * ringbuffer/fifo.
240 * @KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB: Atom not currently present in slot fifo,
241 * which implies that either atom has not become
242 * runnable due to dependency or has completed
243 * the execution on GPU.
244 * @KBASE_ATOM_GPU_RB_WAITING_BLOCKED: Atom has been added to slot fifo but is
245 * blocked due to cross slot dependency,
246 * can't be submitted to GPU.
247 * @KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV: Atom has been added to slot
248 * fifo but is waiting for the completion of
249 * previously added atoms in current & other
250 * slots, as their protected mode requirements
251 * do not match with the current atom.
252 * @KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION: Atom is in slot fifo
253 * and is waiting for completion of protected
254 * mode transition, needed before the atom is
255 * submitted to GPU.
256 * @KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE: Atom is in slot fifo but is
257 * waiting for the cores, which are needed to
258 * execute the job chain represented by the atom,
259 * to become available
260 * @KBASE_ATOM_GPU_RB_READY: Atom is in slot fifo and can be submitted to
261 * GPU.
262 * @KBASE_ATOM_GPU_RB_SUBMITTED: Atom is in slot fifo and has been submitted
263 * to GPU.
264 * @KBASE_ATOM_GPU_RB_RETURN_TO_JS: Atom must be returned to JS due to some
265 * failure, but only after the previously added
266 * atoms in fifo have completed or have also
267 * been returned to JS.
268 */
269 enum kbase_atom_gpu_rb_state {
270 KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB,
271 KBASE_ATOM_GPU_RB_WAITING_BLOCKED,
272 KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV,
273 KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION,
274 KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE,
275 KBASE_ATOM_GPU_RB_READY,
276 KBASE_ATOM_GPU_RB_SUBMITTED,
277 KBASE_ATOM_GPU_RB_RETURN_TO_JS = -1
278 };
279
280 /**
281 * enum kbase_atom_enter_protected_state - The state of an atom with respect to
282 * the preparation for GPU's entry into protected mode,
283 * becomes pertinent only after atom's state with respect
284 * to slot ringbuffer is
285 * KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION
286 * @KBASE_ATOM_ENTER_PROTECTED_CHECK: Starting state. Check if there are any
287 * atoms currently submitted to GPU and protected mode
288 * transition is not already in progress.
289 * @KBASE_ATOM_ENTER_PROTECTED_HWCNT: Wait for hardware counter context to
290 * become disabled before entry into protected mode.
291 * @KBASE_ATOM_ENTER_PROTECTED_IDLE_L2: Wait for the L2 to become idle in
292 * preparation for the coherency change. L2 shall be
293 * powered down and GPU shall come out of fully
294 * coherent mode before entering protected mode.
295 * @KBASE_ATOM_ENTER_PROTECTED_SET_COHERENCY: Prepare coherency change;
296 * for BASE_HW_ISSUE_TGOX_R1_1234 also request L2 power on
297 * so that coherency register contains correct value when
298 * GPU enters protected mode.
299 * @KBASE_ATOM_ENTER_PROTECTED_FINISHED: End state; for
300 * BASE_HW_ISSUE_TGOX_R1_1234 check
301 * that L2 is powered up and switch GPU to protected mode.
302 */
303 enum kbase_atom_enter_protected_state {
304 /*
305 * NOTE: The integer value of this must match
306 * KBASE_ATOM_EXIT_PROTECTED_CHECK.
307 */
308 KBASE_ATOM_ENTER_PROTECTED_CHECK = 0,
309 KBASE_ATOM_ENTER_PROTECTED_HWCNT,
310 KBASE_ATOM_ENTER_PROTECTED_IDLE_L2,
311 KBASE_ATOM_ENTER_PROTECTED_SET_COHERENCY,
312 KBASE_ATOM_ENTER_PROTECTED_FINISHED,
313 };
314
315 /**
316 * enum kbase_atom_exit_protected_state - The state of an atom with respect to
317 * the preparation for GPU's exit from protected mode,
318 * becomes pertinent only after atom's state with respect
319 * to slot ngbuffer is
320 * KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION
321 * @KBASE_ATOM_EXIT_PROTECTED_CHECK: Starting state. Check if there are any
322 * atoms currently submitted to GPU and protected mode
323 * transition is not already in progress.
324 * @KBASE_ATOM_EXIT_PROTECTED_IDLE_L2: Wait for the L2 to become idle in
325 * preparation for the reset, as exiting protected mode
326 * requires a reset.
327 * @KBASE_ATOM_EXIT_PROTECTED_RESET: Issue the reset to trigger exit from
328 * protected mode
329 * @KBASE_ATOM_EXIT_PROTECTED_RESET_WAIT: End state, Wait for the reset to
330 * complete
331 */
332 enum kbase_atom_exit_protected_state {
333 /*
334 * NOTE: The integer value of this must match
335 * KBASE_ATOM_ENTER_PROTECTED_CHECK.
336 */
337 KBASE_ATOM_EXIT_PROTECTED_CHECK = 0,
338 KBASE_ATOM_EXIT_PROTECTED_IDLE_L2,
339 KBASE_ATOM_EXIT_PROTECTED_RESET,
340 KBASE_ATOM_EXIT_PROTECTED_RESET_WAIT,
341 };
342
343 /**
344 * struct kbase_ext_res - Contains the info for external resources referred
345 * by an atom, which have been mapped on GPU side.
346 * @gpu_address: Start address of the memory region allocated for
347 * the resource from GPU virtual address space.
348 * @alloc: pointer to physical pages tracking object, set on
349 * mapping the external resource on GPU side.
350 */
351 struct kbase_ext_res {
352 u64 gpu_address;
353 struct kbase_mem_phy_alloc *alloc;
354 };
355
356 /**
357 * struct kbase_jd_atom - object representing the atom, containing the complete
358 * state and attributes of an atom.
359 * @work: work item for the bottom half processing of the atom,
360 * by JD or JS, after it got executed on GPU or the
361 * input fence got signaled
362 * @start_timestamp: time at which the atom was submitted to the GPU, by
363 * updating the JS_HEAD_NEXTn register.
364 * @udata: copy of the user data sent for the atom in
365 * base_jd_submit.
366 * @kctx: Pointer to the base context with which the atom is
367 * associated.
368 * @dep_head: Array of 2 list heads, pointing to the two list of
369 * atoms
370 * which are blocked due to dependency on this atom.
371 * @dep_item: Array of 2 list heads, used to store the atom in the
372 * list of other atoms depending on the same dependee
373 * atom.
374 * @dep: Array containing the dependency info for the 2 atoms
375 * on which the atom depends upon.
376 * @jd_item: List head used during job dispatch job_done
377 * processing - as dependencies may not be entirely
378 * resolved at this point,
379 * we need to use a separate list head.
380 * @in_jd_list: flag set to true if atom's @jd_item is currently on
381 * a list, prevents atom being processed twice.
382 * @jit_ids: Zero-terminated array of IDs of just-in-time memory
383 * allocations written to by the atom. When the atom
384 * completes, the value stored at the
385 * &struct_base_jit_alloc_info.heap_info_gpu_addr of
386 * each allocation is read in order to enforce an
387 * overall physical memory usage limit.
388 * @nr_extres: number of external resources referenced by the atom.
389 * @extres: pointer to the location containing info about
390 * @nr_extres external resources referenced by the atom.
391 * @device_nr: indicates the coregroup with which the atom is
392 * associated, when
393 * BASE_JD_REQ_SPECIFIC_COHERENT_GROUP specified.
394 * @jc: GPU address of the job-chain.
395 * @softjob_data: Copy of data read from the user space buffer that @jc
396 * points to.
397 * @fence: Stores either an input or output sync fence,
398 * depending on soft-job type
399 * @sync_waiter: Pointer to the sync fence waiter structure passed to
400 * the callback function on signaling of the input
401 * fence.
402 * @dma_fence: object containing pointers to both input & output
403 * fences and other related members used for explicit
404 * sync through soft jobs and for the implicit
405 * synchronization required on access to external
406 * resources.
407 * @dma_fence.fence_in: Input fence
408 * @dma_fence.fence: Points to the dma-buf output fence for this atom.
409 * @dma_fence.context: The dma-buf fence context number for this atom. A
410 * unique context number is allocated to each katom in
411 * the context on context creation.
412 * @dma_fence.seqno: The dma-buf fence sequence number for this atom. This
413 * is increased every time this katom uses dma-buf fence
414 * @dma_fence.callbacks: List of all callbacks set up to wait on other fences
415 * @dma_fence.dep_count: Atomic counter of number of outstandind dma-buf fence
416 * dependencies for this atom.
417 * @event_code: Event code for the job chain represented by the atom,
418 * both HW and low-level SW events are represented by
419 * event codes.
420 * @core_req: bitmask of BASE_JD_REQ_* flags specifying either
421 * Hw or Sw requirements for the job chain represented
422 * by the atom.
423 * @ticks: Number of scheduling ticks for which atom has been
424 * running on the GPU.
425 * @sched_priority: Priority of the atom for Job scheduling, as per the
426 * KBASE_JS_ATOM_SCHED_PRIO_*.
427 * @completed: Wait queue to wait upon for the completion of atom.
428 * @status: Indicates at high level at what stage the atom is in,
429 * as per KBASE_JD_ATOM_STATE_*, that whether it is not
430 * in use or its queued in JD or given to JS or
431 * submitted to Hw or it completed the execution on Hw.
432 * @work_id: used for GPU tracepoints, its a snapshot of the
433 * 'work_id' counter in kbase_jd_context which is
434 * incremented on every call to base_jd_submit.
435 * @slot_nr: Job slot chosen for the atom.
436 * @atom_flags: bitmask of KBASE_KATOM_FLAG* flags capturing the
437 * excat low level state of the atom.
438 * @gpu_rb_state: bitmnask of KBASE_ATOM_GPU_RB_* flags, precisely
439 * tracking atom's state after it has entered
440 * Job scheduler on becoming runnable. Atom
441 * could be blocked due to cross slot dependency
442 * or waiting for the shader cores to become available
443 * or waiting for protected mode transitions to
444 * complete.
445 * @need_cache_flush_cores_retained: flag indicating that manual flush of GPU
446 * cache is needed for the atom and the shader cores
447 * used for atom have been kept on.
448 * @blocked: flag indicating that atom's resubmission to GPU is
449 * blocked till the work item is scheduled to return the
450 * atom to JS.
451 * @seq_nr: user-space sequence number, to order atoms in some
452 * temporal order
453 * @pre_dep: Pointer to atom that this atom has same-slot
454 * dependency on
455 * @post_dep: Pointer to atom that has same-slot dependency on
456 * this atom
457 * @x_pre_dep: Pointer to atom that this atom has cross-slot
458 * dependency on
459 * @x_post_dep: Pointer to atom that has cross-slot dependency on
460 * this atom
461 * @flush_id: The GPU's flush count recorded at the time of
462 * submission,
463 * used for the cache flush optimization
464 * @fault_event: Info for dumping the debug data on Job fault.
465 * @queue: List head used for 4 different purposes :
466 * Adds atom to the list of dma-buf fence waiting atoms.
467 * Adds atom to the list of atoms blocked due to cross
468 * slot dependency.
469 * Adds atom to the list of softjob atoms for which JIT
470 * allocation has been deferred
471 * Adds atom to the list of softjob atoms waiting for
472 * the signaling of fence.
473 * @jit_node: Used to keep track of all JIT free/alloc jobs in
474 * submission order
475 * @jit_blocked: Flag indicating that JIT allocation requested through
476 * softjob atom will be reattempted after the impending
477 * free of other active JIT allocations.
478 * @will_fail_event_code: If non-zero, this indicates that the atom will fail
479 * with the set event_code when the atom is processed.
480 * Used for special handling of atoms, which have a data
481 * dependency on the failed atoms.
482 * @protected_state: State of the atom, as per
483 * KBASE_ATOM_(ENTER|EXIT)_PROTECTED_*,
484 * when transitioning into or out of protected mode.
485 * Atom will be either entering or exiting the
486 * protected mode.
487 * @protected_state.enter: entering the protected mode.
488 * @protected_state.exit: exiting the protected mode.
489 * @runnable_tree_node: The node added to context's job slot specific rb tree
490 * when the atom becomes runnable.
491 * @age: Age of atom relative to other atoms in the context,
492 * is snapshot of the age_count counter in kbase
493 * context.
494 * @jobslot: Job slot to use when BASE_JD_REQ_JOB_SLOT is specified.
495 * @renderpass_id:Renderpass identifier used to associate an atom that has
496 * BASE_JD_REQ_START_RENDERPASS set in its core requirements
497 * with an atom that has BASE_JD_REQ_END_RENDERPASS set.
498 * @jc_fragment: Set of GPU fragment job chains
499 * @retry_count: TODO: Not used,to be removed
500 */
501 struct kbase_jd_atom {
502 struct work_struct work;
503 ktime_t start_timestamp;
504
505 struct base_jd_udata udata;
506 struct kbase_context *kctx;
507
508 struct list_head dep_head[2];
509 struct list_head dep_item[2];
510 const struct kbase_jd_atom_dependency dep[2];
511 struct list_head jd_item;
512 bool in_jd_list;
513
514 #if MALI_JIT_PRESSURE_LIMIT_BASE
515 u8 jit_ids[2];
516 #endif /* MALI_JIT_PRESSURE_LIMIT_BASE */
517
518 u16 nr_extres;
519 struct kbase_ext_res *extres;
520
521 u32 device_nr;
522 u64 jc;
523 void *softjob_data;
524 #if defined(CONFIG_SYNC)
525 struct sync_fence *fence;
526 struct sync_fence_waiter sync_waiter;
527 #endif /* CONFIG_SYNC */
528 #if defined(CONFIG_MALI_BIFROST_DMA_FENCE) || defined(CONFIG_SYNC_FILE)
529 struct {
530 /* Use the functions/API defined in mali_kbase_fence.h to
531 * when working with this sub struct
532 */
533 #if defined(CONFIG_SYNC_FILE)
534 #if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE)
535 struct fence *fence_in;
536 #else
537 struct dma_fence *fence_in;
538 #endif
539 #endif
540 /* This points to the dma-buf output fence for this atom. If
541 * this is NULL then there is no fence for this atom and the
542 * following fields related to dma_fence may have invalid data.
543 *
544 * The context and seqno fields contain the details for this
545 * fence.
546 *
547 * This fence is signaled when the katom is completed,
548 * regardless of the event_code of the katom (signal also on
549 * failure).
550 */
551 #if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE)
552 struct fence *fence;
553 #else
554 struct dma_fence *fence;
555 #endif
556 unsigned int context;
557 atomic_t seqno;
558 /* This contains a list of all callbacks set up to wait on
559 * other fences. This atom must be held back from JS until all
560 * these callbacks have been called and dep_count have reached
561 * 0. The initial value of dep_count must be equal to the
562 * number of callbacks on this list.
563 *
564 * This list is protected by jctx.lock. Callbacks are added to
565 * this list when the atom is built and the wait are set up.
566 * All the callbacks then stay on the list until all callbacks
567 * have been called and the atom is queued, or cancelled, and
568 * then all callbacks are taken off the list and freed.
569 */
570 struct list_head callbacks;
571 /* Atomic counter of number of outstandind dma-buf fence
572 * dependencies for this atom. When dep_count reaches 0 the
573 * atom may be queued.
574 *
575 * The special value "-1" may only be set after the count
576 * reaches 0, while holding jctx.lock. This indicates that the
577 * atom has been handled, either queued in JS or cancelled.
578 *
579 * If anyone but the dma-fence worker sets this to -1 they must
580 * ensure that any potentially queued worker must have
581 * completed before allowing the atom to be marked as unused.
582 * This can be done by flushing the fence work queue:
583 * kctx->dma_fence.wq.
584 */
585 atomic_t dep_count;
586 } dma_fence;
587 #endif /* CONFIG_MALI_BIFROST_DMA_FENCE || CONFIG_SYNC_FILE */
588
589 /* Note: refer to kbasep_js_atom_retained_state, which will take a copy
590 * of some of the following members
591 */
592 enum base_jd_event_code event_code;
593 base_jd_core_req core_req;
594 u8 jobslot;
595 u8 renderpass_id;
596 struct base_jd_fragment jc_fragment;
597
598 u32 ticks;
599 int sched_priority;
600
601 wait_queue_head_t completed;
602 enum kbase_jd_atom_state status;
603 #if IS_ENABLED(CONFIG_GPU_TRACEPOINTS)
604 int work_id;
605 #endif
606 int slot_nr;
607
608 u32 atom_flags;
609
610 int retry_count;
611
612 enum kbase_atom_gpu_rb_state gpu_rb_state;
613
614 bool need_cache_flush_cores_retained;
615
616 atomic_t blocked;
617
618 u64 seq_nr;
619
620 struct kbase_jd_atom *pre_dep;
621 struct kbase_jd_atom *post_dep;
622
623 struct kbase_jd_atom *x_pre_dep;
624 struct kbase_jd_atom *x_post_dep;
625
626 u32 flush_id;
627
628 #if IS_ENABLED(CONFIG_DEBUG_FS)
629 struct base_job_fault_event fault_event;
630 #endif
631 struct list_head queue;
632
633 struct list_head jit_node;
634 bool jit_blocked;
635
636 enum base_jd_event_code will_fail_event_code;
637
638 union {
639 enum kbase_atom_enter_protected_state enter;
640 enum kbase_atom_exit_protected_state exit;
641 } protected_state;
642
643 struct rb_node runnable_tree_node;
644
645 u32 age;
646 };
647
kbase_jd_katom_is_protected(const struct kbase_jd_atom * katom)648 static inline bool kbase_jd_katom_is_protected(
649 const struct kbase_jd_atom *katom)
650 {
651 return (bool)(katom->atom_flags & KBASE_KATOM_FLAG_PROTECTED);
652 }
653
654 /**
655 * kbase_atom_is_younger - query if one atom is younger by age than another
656 * @katom_a: the first atom
657 * @katom_a: the second atom
658 *
659 * Return: true if the first atom is strictly younger than the second, false
660 * otherwise.
661 */
kbase_jd_atom_is_younger(const struct kbase_jd_atom * katom_a,const struct kbase_jd_atom * katom_b)662 static inline bool kbase_jd_atom_is_younger(const struct kbase_jd_atom *katom_a,
663 const struct kbase_jd_atom *katom_b)
664 {
665 return ((s32)(katom_a->age - katom_b->age) < 0);
666 }
667
668 /**
669 * kbase_jd_atom_is_earlier
670 * @katom_a: the first atom
671 * @katom_b: the second atom
672 *
673 * Return: true if the first atom has been submitted earlier than the
674 * second atom. It is used to understand if an atom that is ready has been
675 * submitted earlier than the currently running atom, so that the currently
676 * running atom should be preempted to allow the ready atom to run.
677 */
kbase_jd_atom_is_earlier(const struct kbase_jd_atom * katom_a,const struct kbase_jd_atom * katom_b)678 static inline bool kbase_jd_atom_is_earlier(const struct kbase_jd_atom *katom_a,
679 const struct kbase_jd_atom *katom_b)
680 {
681 /* No seq_nr set? */
682 if (!katom_a->seq_nr || !katom_b->seq_nr)
683 return false;
684
685 /* Efficiently handle the unlikely case of wrapping.
686 * The following code assumes that the delta between the sequence number
687 * of the two atoms is less than INT64_MAX.
688 * In the extremely unlikely case where the delta is higher, the comparison
689 * defaults for no preemption.
690 * The code also assumes that the conversion from unsigned to signed types
691 * works because the signed integers are 2's complement.
692 */
693 return (s64)(katom_a->seq_nr - katom_b->seq_nr) < 0;
694 }
695
696 /*
697 * Theory of operations:
698 *
699 * Atom objects are statically allocated within the context structure.
700 *
701 * Each atom is the head of two lists, one for the "left" set of dependencies,
702 * one for the "right" set.
703 */
704
705 #define KBASE_JD_DEP_QUEUE_SIZE 256
706
707 /**
708 * enum kbase_jd_renderpass_state - State of a renderpass
709 * @KBASE_JD_RP_COMPLETE: Unused or completed renderpass. Can only transition to
710 * START.
711 * @KBASE_JD_RP_START: Renderpass making a first attempt at tiling.
712 * Can transition to PEND_OOM or COMPLETE.
713 * @KBASE_JD_RP_PEND_OOM: Renderpass whose first attempt at tiling used too much
714 * memory and has a soft-stop pending. Can transition to
715 * OOM or COMPLETE.
716 * @KBASE_JD_RP_OOM: Renderpass whose first attempt at tiling used too much
717 * memory and therefore switched to incremental
718 * rendering. The fragment job chain is forced to run.
719 * Can only transition to RETRY.
720 * @KBASE_JD_RP_RETRY: Renderpass making a second or subsequent attempt at
721 * tiling. Can transition to RETRY_PEND_OOM or COMPLETE.
722 * @KBASE_JD_RP_RETRY_PEND_OOM: Renderpass whose second or subsequent attempt at
723 * tiling used too much memory again and has a
724 * soft-stop pending. Can transition to RETRY_OOM
725 * or COMPLETE.
726 * @KBASE_JD_RP_RETRY_OOM: Renderpass whose second or subsequent attempt at
727 * tiling used too much memory again. The fragment job
728 * chain is forced to run. Can only transition to RETRY.
729 *
730 * A state machine is used to control incremental rendering.
731 */
732 enum kbase_jd_renderpass_state {
733 KBASE_JD_RP_COMPLETE, /* COMPLETE => START */
734 KBASE_JD_RP_START, /* START => PEND_OOM or COMPLETE */
735 KBASE_JD_RP_PEND_OOM, /* PEND_OOM => OOM or COMPLETE */
736 KBASE_JD_RP_OOM, /* OOM => RETRY */
737 KBASE_JD_RP_RETRY, /* RETRY => RETRY_PEND_OOM or
738 * COMPLETE
739 */
740 KBASE_JD_RP_RETRY_PEND_OOM, /* RETRY_PEND_OOM => RETRY_OOM or
741 * COMPLETE
742 */
743 KBASE_JD_RP_RETRY_OOM, /* RETRY_OOM => RETRY */
744 };
745
746 /**
747 * struct kbase_jd_renderpass - Data for a renderpass
748 * @state: Current state of the renderpass. If KBASE_JD_RP_COMPLETE then
749 * all other members are invalid.
750 * Both the job dispatcher context and hwaccess_lock must be
751 * locked to modify this so that it can be read with either
752 * (or both) locked.
753 * @start_katom: Address of the atom that is the start of a renderpass.
754 * Both the job dispatcher context and hwaccess_lock must be
755 * locked to modify this so that it can be read with either
756 * (or both) locked.
757 * @end_katom: Address of the atom that is the end of a renderpass, or NULL
758 * if that atom hasn't been added to the job scheduler yet.
759 * The job dispatcher context and hwaccess_lock must be
760 * locked to modify this so that it can be read with either
761 * (or both) locked.
762 * @oom_reg_list: A list of region structures which triggered out-of-memory.
763 * The hwaccess_lock must be locked to access this.
764 *
765 * Atoms tagged with BASE_JD_REQ_START_RENDERPASS or BASE_JD_REQ_END_RENDERPASS
766 * are associated with an object of this type, which is created and maintained
767 * by kbase to keep track of each renderpass.
768 */
769 struct kbase_jd_renderpass {
770 enum kbase_jd_renderpass_state state;
771 struct kbase_jd_atom *start_katom;
772 struct kbase_jd_atom *end_katom;
773 struct list_head oom_reg_list;
774 };
775
776 /**
777 * struct kbase_jd_context - per context object encapsulating all the
778 * Job dispatcher related state.
779 * @lock: lock to serialize the updates made to the
780 * Job dispatcher state and kbase_jd_atom objects.
781 * @sched_info: Structure encapsulating all the Job scheduling
782 * info.
783 * @atoms: Array of the objects representing atoms,
784 * containing the complete state and attributes
785 * of an atom.
786 * @renderpasses: Array of renderpass state for incremental
787 * rendering, indexed by user-specified renderpass
788 * ID.
789 * @job_nr: Tracks the number of atoms being processed by the
790 * kbase. This includes atoms that are not tracked by
791 * scheduler: 'not ready to run' & 'dependency-only'
792 * jobs.
793 * @zero_jobs_wait: Waitq that reflects whether there are no jobs
794 * (including SW-only dependency jobs). This is set
795 * when no jobs are present on the ctx, and clear
796 * when there are jobs.
797 * This must be updated atomically with @job_nr.
798 * note: Job Dispatcher knows about more jobs than
799 * the Job Scheduler as it is unaware of jobs that
800 * are blocked on dependencies and SW-only dependency
801 * jobs. This waitq can be waited upon to find out
802 * when the context jobs are all done/cancelled
803 * (including those that might've been blocked
804 * on dependencies) - and so, whether it can be
805 * terminated. However, it should only be terminated
806 * once it is not present in the run-pool.
807 * Since the waitq is only set under @lock,
808 * the waiter should also briefly obtain and drop
809 * @lock to guarantee that the setter has completed
810 * its work on the kbase_context
811 * @job_done_wq: Workqueue to which the per atom work item is
812 * queued for bottom half processing when the
813 * atom completes
814 * execution on GPU or the input fence get signaled.
815 * @tb_lock: Lock to serialize the write access made to @tb to
816 * to store the register access trace messages.
817 * @tb: Pointer to the Userspace accessible buffer storing
818 * the trace messages for register read/write
819 * accesses made by the Kbase. The buffer is filled
820 * in circular fashion.
821 * @tb_wrap_offset: Offset to the end location in the trace buffer,
822 * the write pointer is moved to the beginning on
823 * reaching this offset.
824 * @work_id: atomic variable used for GPU tracepoints,
825 * incremented on every call to base_jd_submit.
826 * @jit_atoms_head: A list of the just-in-time memory soft-jobs, both
827 * allocate & free, in submission order, protected
828 * by kbase_jd_context.lock.
829 * @jit_pending_alloc: A list of just-in-time memory allocation
830 * soft-jobs which will be reattempted after the
831 * impending free of other active allocations.
832 * @max_priority: Max priority level allowed for this context.
833 */
834 struct kbase_jd_context {
835 struct mutex lock;
836 struct kbasep_js_kctx_info sched_info;
837 struct kbase_jd_atom atoms[BASE_JD_ATOM_COUNT];
838 struct kbase_jd_renderpass renderpasses[BASE_JD_RP_COUNT];
839 struct workqueue_struct *job_done_wq;
840
841 wait_queue_head_t zero_jobs_wait;
842 spinlock_t tb_lock;
843 u32 *tb;
844 u32 job_nr;
845 size_t tb_wrap_offset;
846
847 #if IS_ENABLED(CONFIG_GPU_TRACEPOINTS)
848 atomic_t work_id;
849 #endif
850
851 struct list_head jit_atoms_head;
852 struct list_head jit_pending_alloc;
853 int max_priority;
854 };
855
856 /**
857 * struct jsctx_queue - JS context atom queue
858 * @runnable_tree: Root of RB-tree containing currently runnable atoms on this
859 * job slot.
860 * @x_dep_head: Head item of the linked list of atoms blocked on cross-slot
861 * dependencies. Atoms on this list will be moved to the
862 * runnable_tree when the blocking atom completes.
863 *
864 * hwaccess_lock must be held when accessing this structure.
865 */
866 struct jsctx_queue {
867 struct rb_root runnable_tree;
868 struct list_head x_dep_head;
869 };
870
871 /**
872 * struct kbase_as - Object representing an address space of GPU.
873 * @number: Index at which this address space structure is present
874 * in an array of address space structures embedded inside
875 * the &struct kbase_device.
876 * @pf_wq: Workqueue for processing work items related to
877 * Page fault and Bus fault handling.
878 * @work_pagefault: Work item for the Page fault handling.
879 * @work_busfault: Work item for the Bus fault handling.
880 * @pf_data: Data relating to Page fault.
881 * @bf_data: Data relating to Bus fault.
882 * @current_setup: Stores the MMU configuration for this address space.
883 */
884 struct kbase_as {
885 int number;
886 struct workqueue_struct *pf_wq;
887 struct work_struct work_pagefault;
888 struct work_struct work_busfault;
889 struct kbase_fault pf_data;
890 struct kbase_fault bf_data;
891 struct kbase_mmu_setup current_setup;
892 };
893
894 #endif /* _KBASE_JM_DEFS_H_ */
895