1 /* 2 * 3 * (C) COPYRIGHT 2018-2020 ARM Limited. All rights reserved. 4 * 5 * This program is free software and is provided to you under the terms of the 6 * GNU General Public License version 2 as published by the Free Software 7 * Foundation, and any use by you of this program is subject to the terms 8 * of such GNU licence. 9 * 10 * This program is distributed in the hope that it will be useful, 11 * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 * GNU General Public License for more details. 14 * 15 * You should have received a copy of the GNU General Public License 16 * along with this program; if not, you can access it online at 17 * http://www.gnu.org/licenses/gpl-2.0.html. 18 * 19 * SPDX-License-Identifier: GPL-2.0 20 * 21 */ 22 23 /* Definitions (types, defines, etcs) common to the command stream frontend. 24 * They are placed here to allow the hierarchy of header files to work. 25 */ 26 27 #ifndef KBASE_CSF_DEFS_H 28 #define KBASE_CSF_DEFS_H 29 30 #include <linux/types.h> 31 #include <linux/wait.h> 32 33 #include "mali_kbase_csf_firmware.h" 34 35 /* Maximum number of KCPU command queues to be created per GPU address space. 36 */ 37 #define KBASEP_MAX_KCPU_QUEUES ((size_t)256) 38 39 /* Maximum number of GPU command queue groups to be created per GPU address 40 * space. 41 */ 42 #define MAX_QUEUE_GROUP_NUM (256) 43 44 /* Maximum number of GPU tiler heaps to allow to be created per GPU address 45 * space. 46 */ 47 #define MAX_TILER_HEAPS (128) 48 49 /** 50 * enum kbase_csf_bind_state - bind state of the queue 51 * 52 * @KBASE_CSF_QUEUE_UNBOUND: Set when the queue is registered or when the link 53 * between queue and the group to which it was bound or being bound is removed. 54 * @KBASE_CSF_QUEUE_BIND_IN_PROGRESS: Set when the first part of bind operation 55 * has completed i.e. CS_QUEUE_BIND ioctl. 56 * @KBASE_CSF_QUEUE_BOUND: Set when the bind operation has completed i.e. IO 57 * pages have been mapped in the process address space. 58 */ 59 enum kbase_csf_queue_bind_state { 60 KBASE_CSF_QUEUE_UNBOUND, 61 KBASE_CSF_QUEUE_BIND_IN_PROGRESS, 62 KBASE_CSF_QUEUE_BOUND, 63 }; 64 65 /** 66 * enum kbase_csf_reset_gpu_state - state of the gpu reset 67 * 68 * @KBASE_CSF_RESET_GPU_NOT_PENDING: Set when the GPU reset isn't pending 69 * @KBASE_CSF_RESET_GPU_HAPPENING: Set when the GPU reset process is occurring 70 * @KBASE_CSF_RESET_GPU_SILENT: Set when the GPU reset process is occurring, 71 * used when resetting the GPU as part of normal behavior (e.g. when exiting 72 * protected mode). 73 * @KBASE_CSF_RESET_GPU_FAILED: Set when an error is encountered during the 74 * GPU reset process. No more work could then be executed on GPU, unloading 75 * the Driver module is the only option. 76 */ 77 enum kbase_csf_reset_gpu_state { 78 KBASE_CSF_RESET_GPU_NOT_PENDING, 79 KBASE_CSF_RESET_GPU_HAPPENING, 80 KBASE_CSF_RESET_GPU_SILENT, 81 KBASE_CSF_RESET_GPU_FAILED, 82 }; 83 84 /** 85 * enum kbase_csf_group_state - state of the GPU command queue group 86 * 87 * @KBASE_CSF_GROUP_INACTIVE: Group is inactive and won't be 88 * considered by scheduler for running on 89 * command stream group slot. 90 * @KBASE_CSF_GROUP_RUNNABLE: Group is in the list of runnable groups 91 * and is subjected to time-slice based 92 * scheduling. A start request would be 93 * sent (or already has been sent) if the 94 * group is assigned the command stream 95 * group slot for the fist time. 96 * @KBASE_CSF_GROUP_IDLE: Group is currently on a command stream 97 * group slot but all the command streams 98 * bound to the group have become either 99 * idle or waiting on sync object. 100 * Group could be evicted from the slot on 101 * the next tick if there are no spare 102 * slots left after scheduling non-idle 103 * queue groups. If the group is kept on 104 * slot then it would be moved to the 105 * RUNNABLE state, also if one of the 106 * queues bound to the group is kicked it 107 * would be moved to the RUNNABLE state. 108 * If the group is evicted from the slot it 109 * would be moved to either 110 * KBASE_CSF_GROUP_SUSPENDED_ON_IDLE or 111 * KBASE_CSF_GROUP_SUSPENDED_ON_WAIT_SYNC 112 * state. 113 * @KBASE_CSF_GROUP_SUSPENDED: Group was evicted from the command 114 * stream group slot and is not running but 115 * is still in the list of runnable groups 116 * and subjected to time-slice based 117 * scheduling. A resume request would be 118 * sent when a command stream group slot is 119 * re-assigned to the group and once the 120 * resume is complete group would be moved 121 * back to the RUNNABLE state. 122 * @KBASE_CSF_GROUP_SUSPENDED_ON_IDLE: Same as KBASE_CSF_GROUP_SUSPENDED except 123 * that queue group also became idle before 124 * the suspension. This state helps 125 * Scheduler avoid scheduling the idle 126 * groups over the non-idle groups in the 127 * subsequent ticks. If one of the queues 128 * bound to the group is kicked it would be 129 * moved to the SUSPENDED state. 130 * @KBASE_CSF_GROUP_SUSPENDED_ON_WAIT_SYNC: Same as GROUP_SUSPENDED_ON_IDLE 131 * except that at least one command 132 * stream bound to this group was 133 * waiting for synchronization object 134 * before the suspension. 135 * @KBASE_CSF_GROUP_FAULT_EVICTED: Group is evicted from the scheduler due 136 * to a fault condition, pending to be 137 * terminated. 138 * @KBASE_CSF_GROUP_TERMINATED: Group is no longer schedulable and is 139 * pending to be deleted by Client, all the 140 * queues bound to it have been unbound. 141 */ 142 enum kbase_csf_group_state { 143 KBASE_CSF_GROUP_INACTIVE, 144 KBASE_CSF_GROUP_RUNNABLE, 145 KBASE_CSF_GROUP_IDLE, 146 KBASE_CSF_GROUP_SUSPENDED, 147 KBASE_CSF_GROUP_SUSPENDED_ON_IDLE, 148 KBASE_CSF_GROUP_SUSPENDED_ON_WAIT_SYNC, 149 KBASE_CSF_GROUP_FAULT_EVICTED, 150 KBASE_CSF_GROUP_TERMINATED, 151 }; 152 153 /** 154 * enum kbase_csf_csg_slot_state - state of the command queue group slots under 155 * the scheduler control. 156 * 157 * @CSG_SLOT_READY: The slot is clean and ready to be programmed with a 158 * queue group. 159 * @CSG_SLOT_READY2RUN: The slot has been programmed with a queue group, i.e. a 160 * start or resume request has been sent to the firmware. 161 * @CSG_SLOT_RUNNING: The queue group is running on the slot, acknowledgment 162 * of a start or resume request has been obtained from the 163 * firmware. 164 * @CSG_SLOT_DOWN2STOP: The suspend or terminate request for the queue group on 165 * the slot has been sent to the firmware. 166 * @CSG_SLOT_STOPPED: The queue group is removed from the slot, acknowledgment 167 * of suspend or terminate request has been obtained from 168 * the firmware. 169 * @CSG_SLOT_READY2RUN_TIMEDOUT: The start or resume request sent on the slot 170 * for the queue group timed out. 171 * @CSG_SLOT_DOWN2STOP_TIMEDOUT: The suspend or terminate request for queue 172 * group on the slot timed out. 173 */ 174 enum kbase_csf_csg_slot_state { 175 CSG_SLOT_READY, 176 CSG_SLOT_READY2RUN, 177 CSG_SLOT_RUNNING, 178 CSG_SLOT_DOWN2STOP, 179 CSG_SLOT_STOPPED, 180 CSG_SLOT_READY2RUN_TIMEDOUT, 181 CSG_SLOT_DOWN2STOP_TIMEDOUT, 182 }; 183 184 /** 185 * enum kbase_csf_scheduler_state - state of the scheduler operational phases. 186 * 187 * @SCHED_BUSY: The scheduler is busy performing on tick schedule 188 * operations, the state of command stream group slots 189 * can't be changed. 190 * @SCHED_INACTIVE: The scheduler is inactive, it is allowed to modify the 191 * state of command stream group slots by in-cycle 192 * priority scheduling. 193 * @SCHED_SUSPENDED: The scheduler is in low-power mode with scheduling 194 * operations suspended and is not holding the power 195 * management reference. This can happen if the GPU 196 * becomes idle for a duration exceeding a threshold, 197 * or due to a system triggered suspend action. 198 */ 199 enum kbase_csf_scheduler_state { 200 SCHED_BUSY, 201 SCHED_INACTIVE, 202 SCHED_SUSPENDED, 203 }; 204 205 /** 206 * struct kbase_csf_notification - Event or error generated as part of command 207 * queue execution 208 * 209 * @data: Event or error data returned to userspace 210 * @link: Link to the linked list, &struct_kbase_csf_context.error_list. 211 */ 212 struct kbase_csf_notification { 213 struct base_csf_notification data; 214 struct list_head link; 215 }; 216 217 /** 218 * struct kbase_queue - Object representing a GPU command queue. 219 * 220 * @kctx: Pointer to the base context with which this GPU command queue 221 * is associated. 222 * @reg: Pointer to the region allocated from the shared 223 * interface segment for mapping the User mode 224 * input/output pages in MCU firmware address space. 225 * @phys: Pointer to the physical pages allocated for the 226 * pair or User mode input/output page 227 * @user_io_addr: Pointer to the permanent kernel mapping of User mode 228 * input/output pages. The pages can be accessed through 229 * the mapping without any cache maintenance. 230 * @handle: Handle returned with bind ioctl for creating a 231 * contiguous User mode mapping of input/output pages & 232 * the hardware doorbell page. 233 * @doorbell_nr: Index of the hardware doorbell page assigned to the 234 * queue. 235 * @db_file_offset: File offset value that is assigned to userspace mapping 236 * created on bind to access the doorbell page. 237 * It is in page units. 238 * @link: Link to the linked list of GPU command queues created per 239 * GPU address space. 240 * @refcount: Reference count, stands for the number of times the queue 241 * has been referenced. The reference is taken when it is 242 * created, when it is bound to the group and also when the 243 * @oom_event_work or @fault_event_work work item is queued 244 * for it. 245 * @group: Pointer to the group to which this queue is bound. 246 * @queue_reg: Pointer to the VA region allocated for command 247 * stream buffer. 248 * @oom_event_work: Work item corresponding to the out of memory event for 249 * chunked tiler heap being used for this queue. 250 * @fault_event_work: Work item corresponding to the firmware fault event. 251 * @base_addr: Base address of the command stream buffer. 252 * @size: Size of the command stream buffer. 253 * @priority: Priority of this queue within the group. 254 * @bind_state: Bind state of the queue. 255 * @csi_index: The ID of the assigned command stream hardware interface. 256 * @enabled: Indicating whether the command stream is running, or not. 257 * @status_wait: Value of CS_STATUS_WAIT register of the command stream will 258 * be kept when the command stream gets blocked by sync wait. 259 * CS_STATUS_WAIT provides information on conditions queue is 260 * blocking on. This is set when the group, to which queue is 261 * bound, is suspended after getting blocked, i.e. in 262 * KBASE_CSF_GROUP_SUSPENDED_ON_WAIT_SYNC state. 263 * @sync_ptr: Value of CS_STATUS_WAIT_SYNC_POINTER register of the command 264 * stream will be kept when the command stream gets blocked by 265 * sync wait. CS_STATUS_WAIT_SYNC_POINTER contains the address 266 * of synchronization object being waited on. 267 * Valid only when @status_wait is set. 268 * @sync_value: Value of CS_STATUS_WAIT_SYNC_VALUE register of the command 269 * stream will be kept when the command stream gets blocked by 270 * sync wait. CS_STATUS_WAIT_SYNC_VALUE contains the value 271 * tested against the synchronization object. 272 * Valid only when @status_wait is set. 273 * @error: GPU command queue fatal information to pass to user space. 274 */ 275 struct kbase_queue { 276 struct kbase_context *kctx; 277 struct kbase_va_region *reg; 278 struct tagged_addr phys[2]; 279 char *user_io_addr; 280 u64 handle; 281 int doorbell_nr; 282 unsigned long db_file_offset; 283 struct list_head link; 284 atomic_t refcount; 285 struct kbase_queue_group *group; 286 struct kbase_va_region *queue_reg; 287 struct work_struct oom_event_work; 288 struct work_struct fault_event_work; 289 u64 base_addr; 290 u32 size; 291 u8 priority; 292 u8 bind_state; 293 s8 csi_index; 294 bool enabled; 295 u32 status_wait; 296 u64 sync_ptr; 297 u32 sync_value; 298 struct kbase_csf_notification error; 299 }; 300 301 /** 302 * struct kbase_normal_suspend_buffer - Object representing a normal 303 * suspend buffer for queue group. 304 * @reg: Memory region allocated for the normal-mode suspend buffer. 305 * @phy: Array of physical memory pages allocated for the normal- 306 * mode suspend buffer. 307 */ 308 struct kbase_normal_suspend_buffer { 309 struct kbase_va_region *reg; 310 struct tagged_addr *phy; 311 }; 312 313 /** 314 * struct kbase_protected_suspend_buffer - Object representing a protected 315 * suspend buffer for queue group. 316 * @reg: Memory region allocated for the protected-mode suspend buffer. 317 * @pma: Array of pointer to protected mode allocations containing 318 * information about memory pages allocated for protected mode 319 * suspend buffer. 320 */ 321 struct kbase_protected_suspend_buffer { 322 struct kbase_va_region *reg; 323 struct protected_memory_allocation **pma; 324 }; 325 326 /** 327 * struct kbase_queue_group - Object representing a GPU command queue group. 328 * 329 * @kctx: Pointer to the kbase context with which this queue group 330 * is associated. 331 * @normal_suspend_buf: Object representing the normal suspend buffer. 332 * Normal-mode suspend buffer that is used for 333 * group context switch. 334 * @protected_suspend_buf: Object representing the protected suspend 335 * buffer. Protected-mode suspend buffer that is 336 * used for group context switch. 337 * @handle: Handle which identifies this queue group. 338 * @csg_nr: Number/index of the command stream group to 339 * which this queue group is mapped; KBASEP_CSG_NR_INVALID 340 * indicates that the queue group is not scheduled. 341 * @priority: Priority of the queue group, 0 being the highest, 342 * BASE_QUEUE_GROUP_PRIORITY_COUNT - 1 being the lowest. 343 * @tiler_max: Maximum number of tiler endpoints the group is allowed 344 * to use. 345 * @fragment_max: Maximum number of fragment endpoints the group is 346 * allowed to use. 347 * @compute_max: Maximum number of compute endpoints the group is 348 * allowed to use. 349 * @tiler_mask: Mask of tiler endpoints the group is allowed to use. 350 * @fragment_mask: Mask of fragment endpoints the group is allowed to use. 351 * @compute_mask: Mask of compute endpoints the group is allowed to use. 352 * @link: Link to this queue group in the 'runnable_groups' list of 353 * the corresponding kctx. 354 * @link_to_schedule: Link to this queue group in the list of prepared groups 355 * to be scheduled, if the group is runnable/suspended. 356 * If the group is idle or waiting for CQS, it would be a 357 * link to the list of idle/blocked groups list. 358 * @timer_event_work: Work item corresponding to the event generated when a task 359 * started by a queue in this group takes too long to execute 360 * on an endpoint. 361 * @run_state: Current state of the queue group. 362 * @prepared_seq_num: Indicates the position of queue group in the list of 363 * prepared groups to be scheduled. 364 * @faulted: Indicates that a GPU fault occurred for the queue group. 365 * This flag persists until the fault has been queued to be 366 * reported to userspace. 367 * @bound_queues: Array of registered queues bound to this queue group. 368 * @doorbell_nr: Index of the hardware doorbell page assigned to the 369 * group. 370 * @protm_event_work: Work item corresponding to the protected mode entry 371 * event for this queue. 372 * @protm_pending_bitmap: Bit array to keep a track of command streams that 373 * have pending protected mode entry requests. 374 * @error_fatal: An error of type BASE_GPU_QUEUE_GROUP_ERROR_FATAL to be 375 * returned to userspace if such an error has occurred. 376 * @error_timeout: An error of type BASE_GPU_QUEUE_GROUP_ERROR_TIMEOUT 377 * to be returned to userspace if such an error has occurred. 378 * @error_tiler_oom: An error of type BASE_GPU_QUEUE_GROUP_ERROR_TILER_HEAP_OOM 379 * to be returned to userspace if such an error has occurred. 380 */ 381 struct kbase_queue_group { 382 struct kbase_context *kctx; 383 struct kbase_normal_suspend_buffer normal_suspend_buf; 384 struct kbase_protected_suspend_buffer protected_suspend_buf; 385 u8 handle; 386 s8 csg_nr; 387 u8 priority; 388 389 u8 tiler_max; 390 u8 fragment_max; 391 u8 compute_max; 392 393 u64 tiler_mask; 394 u64 fragment_mask; 395 u64 compute_mask; 396 397 struct list_head link; 398 struct list_head link_to_schedule; 399 struct work_struct timer_event_work; 400 enum kbase_csf_group_state run_state; 401 u32 prepared_seq_num; 402 bool faulted; 403 404 struct kbase_queue *bound_queues[MAX_SUPPORTED_STREAMS_PER_GROUP]; 405 406 int doorbell_nr; 407 struct work_struct protm_event_work; 408 DECLARE_BITMAP(protm_pending_bitmap, MAX_SUPPORTED_STREAMS_PER_GROUP); 409 410 struct kbase_csf_notification error_fatal; 411 struct kbase_csf_notification error_timeout; 412 struct kbase_csf_notification error_tiler_oom; 413 }; 414 415 /** 416 * struct kbase_csf_kcpu_queue_context - Object representing the kernel CPU 417 * queues for a GPU address space. 418 * 419 * @lock: Lock preventing concurrent access to @array and the @in_use bitmap. 420 * @array: Array of pointers to kernel CPU command queues. 421 * @in_use: Bitmap which indicates which kernel CPU command queues are in use. 422 * @wq: Dedicated workqueue for processing kernel CPU command queues. 423 * @num_cmds: The number of commands that have been enqueued across 424 * all the KCPU command queues. This could be used as a 425 * timestamp to determine the command's enqueueing time. 426 * @jit_cmds_head: A list of the just-in-time memory commands, both 427 * allocate & free, in submission order, protected 428 * by kbase_csf_kcpu_queue_context.lock. 429 * @jit_blocked_queues: A list of KCPU command queues blocked by a pending 430 * just-in-time memory allocation command which will be 431 * reattempted after the impending free of other active 432 * allocations. 433 */ 434 struct kbase_csf_kcpu_queue_context { 435 struct mutex lock; 436 struct kbase_kcpu_command_queue *array[KBASEP_MAX_KCPU_QUEUES]; 437 DECLARE_BITMAP(in_use, KBASEP_MAX_KCPU_QUEUES); 438 struct workqueue_struct *wq; 439 u64 num_cmds; 440 441 struct list_head jit_cmds_head; 442 struct list_head jit_blocked_queues; 443 }; 444 445 /** 446 * struct kbase_csf_heap_context_allocator - Allocator of heap contexts 447 * 448 * Heap context structures are allocated by the kernel for use by the firmware. 449 * The current implementation subdivides a single GPU memory region for use as 450 * a sparse array. 451 * 452 * @kctx: Pointer to the kbase context with which this allocator is 453 * associated. 454 * @region: Pointer to a GPU memory region from which heap context structures 455 * are allocated. NULL if no heap contexts have been allocated. 456 * @gpu_va: GPU virtual address of the start of the region from which heap 457 * context structures are allocated. 0 if no heap contexts have been 458 * allocated. 459 * @lock: Lock preventing concurrent access to the @in_use bitmap. 460 * @in_use: Bitmap that indicates which heap context structures are currently 461 * allocated (in @region). 462 */ 463 struct kbase_csf_heap_context_allocator { 464 struct kbase_context *kctx; 465 struct kbase_va_region *region; 466 u64 gpu_va; 467 struct mutex lock; 468 DECLARE_BITMAP(in_use, MAX_TILER_HEAPS); 469 }; 470 471 /** 472 * struct kbase_csf_tiler_heap_context - Object representing the tiler heaps 473 * context for a GPU address space. 474 * 475 * This contains all of the command-stream front-end state relating to chunked 476 * tiler heaps for one @kbase_context. It is not the same as a heap context 477 * structure allocated by the kernel for use by the firmware. 478 * 479 * @lock: Lock preventing concurrent access to the tiler heaps. 480 * @list: List of tiler heaps. 481 * @ctx_alloc: Allocator for heap context structures. 482 */ 483 struct kbase_csf_tiler_heap_context { 484 struct mutex lock; 485 struct list_head list; 486 struct kbase_csf_heap_context_allocator ctx_alloc; 487 }; 488 489 /** 490 * struct kbase_csf_scheduler_context - Object representing the scheduler's 491 * context for a GPU address space. 492 * 493 * @runnable_groups: Lists of runnable GPU command queue groups in the kctx, 494 * one per queue group priority level. 495 * @num_runnable_grps: Total number of runnable groups across all priority 496 * levels in @runnable_groups. 497 * @idle_wait_groups: A list of GPU command queue groups in which all enabled 498 * GPU command queues are idle and at least one of them 499 * is blocked on a sync wait operation. 500 * @num_idle_wait_grps: Length of the @idle_wait_groups list. 501 * @sync_update_wq: Dedicated workqueue to process work items corresponding 502 * to the sync_update events by sync_set/sync_add 503 * instruction execution on command streams bound to groups 504 * of @idle_wait_groups list. 505 * @sync_update_work: work item to process the sync_update events by 506 * sync_set / sync_add instruction execution on command 507 * streams bound to groups of @idle_wait_groups list. 508 * @ngrp_to_schedule: Number of groups added for the context to the 509 * 'groups_to_schedule' list of scheduler instance. 510 */ 511 struct kbase_csf_scheduler_context { 512 struct list_head runnable_groups[BASE_QUEUE_GROUP_PRIORITY_COUNT]; 513 u32 num_runnable_grps; 514 struct list_head idle_wait_groups; 515 u32 num_idle_wait_grps; 516 struct workqueue_struct *sync_update_wq; 517 struct work_struct sync_update_work; 518 u32 ngrp_to_schedule; 519 }; 520 521 /** 522 * struct kbase_csf_context - Object representing command-stream front-end 523 * for a GPU address space. 524 * 525 * @event_pages_head: A list of pages allocated for the event memory used by 526 * the synchronization objects. A separate list would help 527 * in the fast lookup, since the list is expected to be short 528 * as one page would provide the memory for up to 1K 529 * synchronization objects. 530 * KBASE_PERMANENTLY_MAPPED_MEM_LIMIT_PAGES is the upper 531 * bound on the size of event memory. 532 * @cookies: Bitmask containing of KBASE_CSF_NUM_USER_IO_PAGES_HANDLE 533 * bits, used for creating the User mode CPU mapping in a 534 * deferred manner of a pair of User mode input/output pages 535 * & a hardware doorbell page. 536 * The pages are allocated when a GPU command queue is 537 * bound to a command stream group in kbase_csf_queue_bind. 538 * This helps returning unique handles to Userspace from 539 * kbase_csf_queue_bind and later retrieving the pointer to 540 * queue in the mmap handler. 541 * @user_pages_info: Array containing pointers to queue 542 * structures, used in conjunction with cookies bitmask for 543 * providing a mechansim to create a CPU mapping of 544 * input/output pages & hardware doorbell page. 545 * @lock: Serializes accesses to all members, except for ones that 546 * have their own locks. 547 * @queue_groups: Array of registered GPU command queue groups. 548 * @queue_list: Linked list of GPU command queues not yet deregistered. 549 * Note that queues can persist after deregistration if the 550 * userspace mapping created for them on bind operation 551 * hasn't been removed. 552 * @kcpu_queues: Kernel CPU command queues. 553 * @event_lock: Lock protecting access to @event_callback_list 554 * @event_callback_list: List of callbacks which are registered to serve CSF 555 * events. 556 * @tiler_heaps: Chunked tiler memory heaps. 557 * @wq: Dedicated workqueue to process work items corresponding 558 * to the OoM events raised for chunked tiler heaps being 559 * used by GPU command queues, and progress timeout events. 560 * @link: Link to this csf context in the 'runnable_kctxs' list of 561 * the scheduler instance 562 * @user_reg_vma: Pointer to the vma corresponding to the virtual mapping 563 * of the USER register page. Currently used only for sanity 564 * checking. 565 * @sched: Object representing the scheduler's context 566 * @error_list: List for command stream fatal errors in this context. 567 * Link of fatal error is 568 * &struct_kbase_csf_notification.link. 569 * @lock needs to be held to access to this list. 570 */ 571 struct kbase_csf_context { 572 struct list_head event_pages_head; 573 DECLARE_BITMAP(cookies, KBASE_CSF_NUM_USER_IO_PAGES_HANDLE); 574 struct kbase_queue *user_pages_info[KBASE_CSF_NUM_USER_IO_PAGES_HANDLE]; 575 struct mutex lock; 576 struct kbase_queue_group *queue_groups[MAX_QUEUE_GROUP_NUM]; 577 struct list_head queue_list; 578 struct kbase_csf_kcpu_queue_context kcpu_queues; 579 spinlock_t event_lock; 580 struct list_head event_callback_list; 581 struct kbase_csf_tiler_heap_context tiler_heaps; 582 struct workqueue_struct *wq; 583 struct list_head link; 584 struct vm_area_struct *user_reg_vma; 585 struct kbase_csf_scheduler_context sched; 586 struct list_head error_list; 587 }; 588 589 /** 590 * struct kbase_csf_reset_gpu - Object containing the members required for 591 * GPU reset handling. 592 * @workq: Workqueue to execute the GPU reset work item @work. 593 * @work: Work item for performing the GPU reset. 594 * @wait: Wait queue used to wait for the GPU reset completion. 595 * @state: Tracks if the GPU reset is in progress or not. 596 */ 597 struct kbase_csf_reset_gpu { 598 struct workqueue_struct *workq; 599 struct work_struct work; 600 wait_queue_head_t wait; 601 atomic_t state; 602 }; 603 604 /** 605 * struct kbase_csf_csg_slot - Object containing members for tracking the state 606 * of command stream group slots. 607 * @resident_group: pointer to the queue group that is resident on the 608 * command stream group slot. 609 * @state: state of the slot as per enum kbase_csf_csg_slot_state. 610 * @trigger_jiffies: value of jiffies when change in slot state is recorded. 611 * @priority: dynamic priority assigned to command stream group slot. 612 */ 613 struct kbase_csf_csg_slot { 614 struct kbase_queue_group *resident_group; 615 atomic_t state; 616 unsigned long trigger_jiffies; 617 u8 priority; 618 }; 619 620 /** 621 * struct kbase_csf_scheduler - Object representing the scheduler used for 622 * command-stream front-end for an instance of 623 * GPU platform device. 624 * @lock: Lock to serialize the scheduler operations and 625 * access to the data members. 626 * @interrupt_lock: Lock to protect members accessed by interrupt 627 * handler. 628 * @state: The operational phase the scheduler is in. Primarily 629 * used for indicating what in-cycle schedule actions 630 * are allowed. 631 * @doorbell_inuse_bitmap: Bitmap of hardware doorbell pages keeping track of 632 * which pages are currently available for assignment 633 * to clients. 634 * @csg_inuse_bitmap: Bitmap to keep a track of command stream group slots 635 * that are currently in use. 636 * @csg_slots: The array for tracking the state of command stream 637 * group slots. 638 * @runnable_kctxs: List of Kbase contexts that have runnable command 639 * queue groups. 640 * @groups_to_schedule: List of runnable queue groups prepared on every 641 * scheduler tick. The dynamic priority of the command 642 * stream group slot assigned to a group will depend 643 * upon the position of group in the list. 644 * @ngrp_to_schedule: Number of groups in the @groups_to_schedule list, 645 * incremented when a group is added to the list, used 646 * to record the position of group in the list. 647 * @num_active_address_spaces: Number of GPU address space slots that would get 648 * used to program the groups in @groups_to_schedule 649 * list on all the available command stream group 650 * slots. 651 * @num_csg_slots_for_tick: Number of command stream group slots that can be 652 * active in the given tick/tock. This depends on the 653 * value of @num_active_address_spaces. 654 * @idle_groups_to_schedule: List of runnable queue groups, in which all GPU 655 * command queues became idle or are waiting for 656 * synchronization object, prepared on every 657 * scheduler tick. The groups in this list are 658 * appended to the tail of @groups_to_schedule list 659 * after the scan out so that the idle groups aren't 660 * preferred for scheduling over the non-idle ones. 661 * @total_runnable_grps: Total number of runnable groups across all KCTXs. 662 * @csgs_events_enable_mask: Use for temporary masking off asynchronous events 663 * from firmware (such as OoM events) before a group 664 * is suspended. 665 * @csg_slots_idle_mask: Bit array for storing the mask of command stream 666 * group slots for which idle notification was 667 * received. 668 * @csg_slots_prio_update: Bit array for tracking slots that have an on-slot 669 * priority update operation. 670 * @last_schedule: Time in jiffies recorded when the last "tick" or 671 * "tock" schedule operation concluded. Used for 672 * evaluating the exclusion window for in-cycle 673 * schedule operation. 674 * @timer_enabled: Whether the CSF scheduler wakes itself up for 675 * periodic scheduling tasks. If this value is 0 676 * then it will only perform scheduling under the 677 * influence of external factors e.g., IRQs, IOCTLs. 678 * @wq: Dedicated workqueue to execute the @tick_work. 679 * @tick_work: Work item that would perform the schedule on tick 680 * operation to implement the time slice based 681 * scheduling. 682 * @tock_work: Work item that would perform the schedule on tock 683 * operation to implement the asynchronous scheduling. 684 * @ping_work: Work item that would ping the firmware at regular 685 * intervals, only if there is a single active command 686 * stream group slot, to check if firmware is alive 687 * and would initiate a reset if the ping request 688 * isn't acknowledged. 689 * @top_ctx: Pointer to the Kbase context corresponding to the 690 * @top_grp. 691 * @top_grp: Pointer to queue group inside @groups_to_schedule 692 * list that was assigned the highest slot priority. 693 * @head_slot_priority: The dynamic slot priority to be used for the 694 * queue group at the head of @groups_to_schedule 695 * list. Once the queue group is assigned a command 696 * stream group slot, it is removed from the list and 697 * priority is decremented. 698 * @tock_pending_request: A "tock" request is pending: a group that is not 699 * currently on the GPU demands to be scheduled. 700 * @active_protm_grp: Indicates if firmware has been permitted to let GPU 701 * enter protected mode with the given group. On exit 702 * from protected mode the pointer is reset to NULL. 703 * @gpu_idle_work: Work item for facilitating the scheduler to bring 704 * the GPU to a low-power mode on becoming idle. 705 * @non_idle_suspended_grps: Count of suspended queue groups not idle. 706 * @pm_active_count: Count indicating if the scheduler is owning a power 707 * management reference count. Reference is taken when 708 * the count becomes 1 and is dropped when the count 709 * becomes 0. It is used to enable the power up of MCU 710 * after GPU and L2 cache have been powered up. So when 711 * this count is zero, MCU will not be powered up. 712 */ 713 struct kbase_csf_scheduler { 714 struct mutex lock; 715 spinlock_t interrupt_lock; 716 enum kbase_csf_scheduler_state state; 717 DECLARE_BITMAP(doorbell_inuse_bitmap, CSF_NUM_DOORBELL); 718 DECLARE_BITMAP(csg_inuse_bitmap, MAX_SUPPORTED_CSGS); 719 struct kbase_csf_csg_slot *csg_slots; 720 struct list_head runnable_kctxs; 721 struct list_head groups_to_schedule; 722 u32 ngrp_to_schedule; 723 u32 num_active_address_spaces; 724 u32 num_csg_slots_for_tick; 725 struct list_head idle_groups_to_schedule; 726 u32 total_runnable_grps; 727 DECLARE_BITMAP(csgs_events_enable_mask, MAX_SUPPORTED_CSGS); 728 DECLARE_BITMAP(csg_slots_idle_mask, MAX_SUPPORTED_CSGS); 729 DECLARE_BITMAP(csg_slots_prio_update, MAX_SUPPORTED_CSGS); 730 unsigned long last_schedule; 731 bool timer_enabled; 732 struct workqueue_struct *wq; 733 struct delayed_work tick_work; 734 struct delayed_work tock_work; 735 struct delayed_work ping_work; 736 struct kbase_context *top_ctx; 737 struct kbase_queue_group *top_grp; 738 u8 head_slot_priority; 739 bool tock_pending_request; 740 struct kbase_queue_group *active_protm_grp; 741 struct delayed_work gpu_idle_work; 742 atomic_t non_idle_suspended_grps; 743 u32 pm_active_count; 744 }; 745 746 /** 747 * Number of GPU cycles per unit of the global progress timeout. 748 */ 749 #define GLB_PROGRESS_TIMER_TIMEOUT_SCALE ((u64)1024) 750 751 /** 752 * Maximum value of the global progress timeout. 753 */ 754 #define GLB_PROGRESS_TIMER_TIMEOUT_MAX \ 755 ((GLB_PROGRESS_TIMER_TIMEOUT_MASK >> GLB_PROGRESS_TIMER_TIMEOUT_SHIFT) * GLB_PROGRESS_TIMER_TIMEOUT_SCALE) 756 757 /** 758 * struct kbase_csf - Object representing command-stream front-end for an 759 * instance of GPU platform device. 760 * 761 * @mcu_mmu: MMU page tables for the MCU firmware 762 * @firmware_interfaces: List of interfaces defined in the firmware image 763 * @firmware_config: List of configuration options within the firmware 764 * image 765 * @firmware_timeline_metadata: List of timeline meta-data within the firmware 766 * image 767 * @fw_cfg_kobj: Pointer to the kobject corresponding to the sysf 768 * directory that contains a sub-directory for each 769 * of the configuration option present in the 770 * firmware image. 771 * @firmware_trace_buffers: List of trace buffers described in the firmware 772 * image. 773 * @shared_interface: Pointer to the interface object containing info for 774 * the memory area shared between firmware & host. 775 * @shared_reg_rbtree: RB tree of the memory regions allocated from the 776 * shared interface segment in MCU firmware address 777 * space. 778 * @db_filp: Pointer to a dummy file, that alongwith 779 * @db_file_offsets, facilitates the use of unqiue 780 * file offset for the userspace mapping created 781 * for Hw Doorbell pages. The userspace mapping 782 * is made to point to this file inside the mmap 783 * handler. 784 * @db_file_offsets: Counter that is incremented every time a GPU 785 * command queue is bound to provide a unique file 786 * offset range for @db_filp file, so that pte of 787 * Doorbell page can be zapped through the kernel 788 * function unmap_mapping_range(). It is incremented 789 * in page units. 790 * @dummy_db_page: Address of the dummy page that is mapped in place 791 * of the real Hw doorbell page for the active GPU 792 * command queues after they are stopped or after the 793 * GPU is powered down. 794 * @reg_lock: Lock to serialize the MCU firmware related actions 795 * that affect all contexts such as allocation of 796 * regions from shared interface area, assignment of 797 * of hardware doorbell pages, assignment of CSGs, 798 * sending global requests. 799 * @event_wait: Wait queue to wait for receiving csf events, i.e. 800 * the interrupt from CSF firmware, or scheduler state 801 * changes. 802 * @interrupt_received: Flag set when the interrupt is received from CSF fw 803 * @global_iface: The result of parsing the global interface 804 * structure set up by the firmware, including the 805 * CSGs, CSs, and their properties 806 * @scheduler: The command stream scheduler instance. 807 * @reset: Contain members required for GPU reset handling. 808 * @progress_timeout: Maximum number of GPU clock cycles without forward 809 * progress to allow, for all tasks running on 810 * hardware endpoints (e.g. shader cores), before 811 * terminating a GPU command queue group. 812 * Must not exceed @GLB_PROGRESS_TIMER_TIMEOUT_MAX. 813 * @pma_dev: Pointer to protected memory allocator device. 814 * @firmware_inited: Flag for indicating that the cold-boot stage of 815 * the MCU has completed. 816 * @firmware_reloaded: Flag for indicating a firmware reload operation 817 * in GPU reset has completed. 818 * @firmware_reload_needed: Flag for indicating that the firmware needs to be 819 * reloaded as part of the GPU reset action. 820 * @firmware_reload_work: Work item for facilitating the procedural actions 821 * on reloading the firmware. 822 * @glb_init_request_pending: Flag to indicate that Global requests have been 823 * sent to the FW after MCU was re-enabled and their 824 * acknowledgement is pending. 825 */ 826 struct kbase_csf_device { 827 struct kbase_mmu_table mcu_mmu; 828 struct list_head firmware_interfaces; 829 struct list_head firmware_config; 830 struct list_head firmware_timeline_metadata; 831 struct kobject *fw_cfg_kobj; 832 struct kbase_csf_trace_buffers firmware_trace_buffers; 833 void *shared_interface; 834 struct rb_root shared_reg_rbtree; 835 struct file *db_filp; 836 u32 db_file_offsets; 837 struct tagged_addr dummy_db_page; 838 struct mutex reg_lock; 839 wait_queue_head_t event_wait; 840 bool interrupt_received; 841 struct kbase_csf_global_iface global_iface; 842 struct kbase_csf_scheduler scheduler; 843 struct kbase_csf_reset_gpu reset; 844 atomic64_t progress_timeout; 845 struct protected_memory_allocator_device *pma_dev; 846 bool firmware_inited; 847 bool firmware_reloaded; 848 bool firmware_reload_needed; 849 struct work_struct firmware_reload_work; 850 bool glb_init_request_pending; 851 }; 852 853 /** 854 * struct kbase_as - Object representing an address space of GPU. 855 * @number: Index at which this address space structure is present 856 * in an array of address space structures embedded inside 857 * the &struct kbase_device. 858 * @pf_wq: Workqueue for processing work items related to 859 * Page fault, Bus fault and GPU fault handling. 860 * @work_pagefault: Work item for the Page fault handling. 861 * @work_busfault: Work item for the Bus fault handling. 862 * @work_gpufault: Work item for the GPU fault handling. 863 * @pf_data: Data relating to Page fault. 864 * @bf_data: Data relating to Bus fault. 865 * @gf_data: Data relating to GPU fault. 866 * @current_setup: Stores the MMU configuration for this address space. 867 */ 868 struct kbase_as { 869 int number; 870 struct workqueue_struct *pf_wq; 871 struct work_struct work_pagefault; 872 struct work_struct work_busfault; 873 struct work_struct work_gpufault; 874 struct kbase_fault pf_data; 875 struct kbase_fault bf_data; 876 struct kbase_fault gf_data; 877 struct kbase_mmu_setup current_setup; 878 }; 879 880 #endif /* _KBASE_CSF_DEFS_H_ */ 881