1 /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ 2 /* 3 * 4 * (C) COPYRIGHT 2018-2021 ARM Limited. All rights reserved. 5 * 6 * This program is free software and is provided to you under the terms of the 7 * GNU General Public License version 2 as published by the Free Software 8 * Foundation, and any use by you of this program is subject to the terms 9 * of such GNU license. 10 * 11 * This program is distributed in the hope that it will be useful, 12 * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 * GNU General Public License for more details. 15 * 16 * You should have received a copy of the GNU General Public License 17 * along with this program; if not, you can access it online at 18 * http://www.gnu.org/licenses/gpl-2.0.html. 19 * 20 */ 21 22 /* Definitions (types, defines, etcs) common to the CSF. 23 * They are placed here to allow the hierarchy of header files to work. 24 */ 25 26 #ifndef _KBASE_CSF_DEFS_H_ 27 #define _KBASE_CSF_DEFS_H_ 28 29 #include <linux/types.h> 30 #include <linux/wait.h> 31 32 #include "mali_kbase_csf_firmware.h" 33 #include "mali_kbase_csf_event.h" 34 35 /* Maximum number of KCPU command queues to be created per GPU address space. 36 */ 37 #define KBASEP_MAX_KCPU_QUEUES ((size_t)256) 38 39 /* Maximum number of GPU command queue groups to be created per GPU address 40 * space. 41 */ 42 #define MAX_QUEUE_GROUP_NUM (256) 43 44 /* Maximum number of GPU tiler heaps to allow to be created per GPU address 45 * space. 46 */ 47 #define MAX_TILER_HEAPS (128) 48 49 #define CSF_FIRMWARE_ENTRY_READ (1ul << 0) 50 #define CSF_FIRMWARE_ENTRY_WRITE (1ul << 1) 51 #define CSF_FIRMWARE_ENTRY_EXECUTE (1ul << 2) 52 #define CSF_FIRMWARE_ENTRY_CACHE_MODE (3ul << 3) 53 #define CSF_FIRMWARE_ENTRY_PROTECTED (1ul << 5) 54 #define CSF_FIRMWARE_ENTRY_SHARED (1ul << 30) 55 #define CSF_FIRMWARE_ENTRY_ZERO (1ul << 31) 56 57 /** 58 * enum kbase_csf_bind_state - bind state of the queue 59 * 60 * @KBASE_CSF_QUEUE_UNBOUND: Set when the queue is registered or when the link 61 * between queue and the group to which it was bound or being bound is removed. 62 * @KBASE_CSF_QUEUE_BIND_IN_PROGRESS: Set when the first part of bind operation 63 * has completed i.e. CS_QUEUE_BIND ioctl. 64 * @KBASE_CSF_QUEUE_BOUND: Set when the bind operation has completed i.e. IO 65 * pages have been mapped in the process address space. 66 */ 67 enum kbase_csf_queue_bind_state { 68 KBASE_CSF_QUEUE_UNBOUND, 69 KBASE_CSF_QUEUE_BIND_IN_PROGRESS, 70 KBASE_CSF_QUEUE_BOUND, 71 }; 72 73 /** 74 * enum kbase_csf_reset_gpu_state - state of the gpu reset 75 * 76 * @KBASE_CSF_RESET_GPU_NOT_PENDING: Set when the GPU reset isn't pending 77 * 78 * @KBASE_CSF_RESET_GPU_PREPARED: Set when kbase_prepare_to_reset_gpu() has 79 * been called. This is just for debugging checks to encourage callers to call 80 * kbase_prepare_to_reset_gpu() before kbase_reset_gpu(). 81 * 82 * @KBASE_CSF_RESET_GPU_COMMITTED: Set when the GPU reset process has been 83 * committed and so will definitely happen, but the procedure to reset the GPU 84 * has not yet begun. Other threads must finish accessing the HW before we 85 * reach %KBASE_CSF_RESET_GPU_HAPPENING. 86 * 87 * @KBASE_CSF_RESET_GPU_HAPPENING: Set when the GPU reset process is occurring 88 * (silent or otherwise), and is actively accessing the HW. Any changes to the 89 * HW in other threads might get lost, overridden, or corrupted. 90 * 91 * @KBASE_CSF_RESET_GPU_COMMITTED_SILENT: Set when the GPU reset process has 92 * been committed but has not started happening. This is used when resetting 93 * the GPU as part of normal behavior (e.g. when exiting protected mode). 94 * Other threads must finish accessing the HW before we reach 95 * %KBASE_CSF_RESET_GPU_HAPPENING. 96 * 97 * @KBASE_CSF_RESET_GPU_FAILED: Set when an error is encountered during the 98 * GPU reset process. No more work could then be executed on GPU, unloading 99 * the Driver module is the only option. 100 */ 101 enum kbase_csf_reset_gpu_state { 102 KBASE_CSF_RESET_GPU_NOT_PENDING, 103 KBASE_CSF_RESET_GPU_PREPARED, 104 KBASE_CSF_RESET_GPU_COMMITTED, 105 KBASE_CSF_RESET_GPU_HAPPENING, 106 KBASE_CSF_RESET_GPU_COMMITTED_SILENT, 107 KBASE_CSF_RESET_GPU_FAILED, 108 }; 109 110 /** 111 * enum kbase_csf_group_state - state of the GPU command queue group 112 * 113 * @KBASE_CSF_GROUP_INACTIVE: Group is inactive and won't be 114 * considered by scheduler for running on 115 * CSG slot. 116 * @KBASE_CSF_GROUP_RUNNABLE: Group is in the list of runnable groups 117 * and is subjected to time-slice based 118 * scheduling. A start request would be 119 * sent (or already has been sent) if the 120 * group is assigned the CS 121 * group slot for the fist time. 122 * @KBASE_CSF_GROUP_IDLE: Group is currently on a CSG slot 123 * but all the CSs bound to the group have 124 * become either idle or waiting on sync 125 * object. 126 * Group could be evicted from the slot on 127 * the next tick if there are no spare 128 * slots left after scheduling non-idle 129 * queue groups. If the group is kept on 130 * slot then it would be moved to the 131 * RUNNABLE state, also if one of the 132 * queues bound to the group is kicked it 133 * would be moved to the RUNNABLE state. 134 * If the group is evicted from the slot it 135 * would be moved to either 136 * KBASE_CSF_GROUP_SUSPENDED_ON_IDLE or 137 * KBASE_CSF_GROUP_SUSPENDED_ON_WAIT_SYNC 138 * state. 139 * @KBASE_CSF_GROUP_SUSPENDED: Group was evicted from the CSG slot 140 * and is not running but is still in the 141 * list of runnable groups and subjected 142 * to time-slice based scheduling. A resume 143 * request would be sent when a CSG slot is 144 * re-assigned to the group and once the 145 * resume is complete group would be moved 146 * back to the RUNNABLE state. 147 * @KBASE_CSF_GROUP_SUSPENDED_ON_IDLE: Same as KBASE_CSF_GROUP_SUSPENDED except 148 * that queue group also became idle before 149 * the suspension. This state helps 150 * Scheduler avoid scheduling the idle 151 * groups over the non-idle groups in the 152 * subsequent ticks. If one of the queues 153 * bound to the group is kicked it would be 154 * moved to the SUSPENDED state. 155 * @KBASE_CSF_GROUP_SUSPENDED_ON_WAIT_SYNC: Same as GROUP_SUSPENDED_ON_IDLE 156 * except that at least one CS 157 * bound to this group was 158 * waiting for synchronization object 159 * before the suspension. 160 * @KBASE_CSF_GROUP_FAULT_EVICTED: Group is evicted from the scheduler due 161 * to a fault condition, pending to be 162 * terminated. 163 * @KBASE_CSF_GROUP_TERMINATED: Group is no longer schedulable and is 164 * pending to be deleted by Client, all the 165 * queues bound to it have been unbound. 166 */ 167 enum kbase_csf_group_state { 168 KBASE_CSF_GROUP_INACTIVE, 169 KBASE_CSF_GROUP_RUNNABLE, 170 KBASE_CSF_GROUP_IDLE, 171 KBASE_CSF_GROUP_SUSPENDED, 172 KBASE_CSF_GROUP_SUSPENDED_ON_IDLE, 173 KBASE_CSF_GROUP_SUSPENDED_ON_WAIT_SYNC, 174 KBASE_CSF_GROUP_FAULT_EVICTED, 175 KBASE_CSF_GROUP_TERMINATED, 176 }; 177 178 /** 179 * enum kbase_csf_csg_slot_state - state of the command queue group slots under 180 * the scheduler control. 181 * 182 * @CSG_SLOT_READY: The slot is clean and ready to be programmed with a 183 * queue group. 184 * @CSG_SLOT_READY2RUN: The slot has been programmed with a queue group, i.e. a 185 * start or resume request has been sent to the firmware. 186 * @CSG_SLOT_RUNNING: The queue group is running on the slot, acknowledgment 187 * of a start or resume request has been obtained from the 188 * firmware. 189 * @CSG_SLOT_DOWN2STOP: The suspend or terminate request for the queue group on 190 * the slot has been sent to the firmware. 191 * @CSG_SLOT_STOPPED: The queue group is removed from the slot, acknowledgment 192 * of suspend or terminate request has been obtained from 193 * the firmware. 194 * @CSG_SLOT_READY2RUN_TIMEDOUT: The start or resume request sent on the slot 195 * for the queue group timed out. 196 * @CSG_SLOT_DOWN2STOP_TIMEDOUT: The suspend or terminate request for queue 197 * group on the slot timed out. 198 */ 199 enum kbase_csf_csg_slot_state { 200 CSG_SLOT_READY, 201 CSG_SLOT_READY2RUN, 202 CSG_SLOT_RUNNING, 203 CSG_SLOT_DOWN2STOP, 204 CSG_SLOT_STOPPED, 205 CSG_SLOT_READY2RUN_TIMEDOUT, 206 CSG_SLOT_DOWN2STOP_TIMEDOUT, 207 }; 208 209 /** 210 * enum kbase_csf_scheduler_state - state of the scheduler operational phases. 211 * 212 * @SCHED_BUSY: The scheduler is busy performing on tick schedule 213 * operations, the state of CSG slots 214 * can't be changed. 215 * @SCHED_INACTIVE: The scheduler is inactive, it is allowed to modify the 216 * state of CSG slots by in-cycle 217 * priority scheduling. 218 * @SCHED_SUSPENDED: The scheduler is in low-power mode with scheduling 219 * operations suspended and is not holding the power 220 * management reference. This can happen if the GPU 221 * becomes idle for a duration exceeding a threshold, 222 * or due to a system triggered suspend action. 223 * @SCHED_SLEEPING: The scheduler is in low-power mode with scheduling 224 * operations suspended and is not holding the power 225 * management reference. This state is set, only for the 226 * GPUs that supports the sleep feature, when GPU idle 227 * notification is received. The state is changed to 228 * @SCHED_SUSPENDED from the runtime suspend callback 229 * function after the suspend of CSGs. 230 */ 231 enum kbase_csf_scheduler_state { 232 SCHED_BUSY, 233 SCHED_INACTIVE, 234 SCHED_SUSPENDED, 235 SCHED_SLEEPING, 236 }; 237 238 /** 239 * enum kbase_queue_group_priority - Kbase internal relative priority list. 240 * 241 * @KBASE_QUEUE_GROUP_PRIORITY_REALTIME: The realtime queue group priority. 242 * @KBASE_QUEUE_GROUP_PRIORITY_HIGH: The high queue group priority. 243 * @KBASE_QUEUE_GROUP_PRIORITY_MEDIUM: The medium queue group priority. 244 * @KBASE_QUEUE_GROUP_PRIORITY_LOW: The low queue group priority. 245 * @KBASE_QUEUE_GROUP_PRIORITY_COUNT: The number of priority levels. 246 */ 247 enum kbase_queue_group_priority { 248 KBASE_QUEUE_GROUP_PRIORITY_REALTIME = 0, 249 KBASE_QUEUE_GROUP_PRIORITY_HIGH, 250 KBASE_QUEUE_GROUP_PRIORITY_MEDIUM, 251 KBASE_QUEUE_GROUP_PRIORITY_LOW, 252 KBASE_QUEUE_GROUP_PRIORITY_COUNT 253 }; 254 255 256 /** 257 * struct kbase_csf_notification - Event or error generated as part of command 258 * queue execution 259 * 260 * @data: Event or error data returned to userspace 261 * @link: Link to the linked list, &struct_kbase_csf_context.error_list. 262 */ 263 struct kbase_csf_notification { 264 struct base_csf_notification data; 265 struct list_head link; 266 }; 267 268 /** 269 * struct kbase_queue - Object representing a GPU command queue. 270 * 271 * @kctx: Pointer to the base context with which this GPU command queue 272 * is associated. 273 * @reg: Pointer to the region allocated from the shared 274 * interface segment for mapping the User mode 275 * input/output pages in MCU firmware address space. 276 * @phys: Pointer to the physical pages allocated for the 277 * pair or User mode input/output page 278 * @user_io_addr: Pointer to the permanent kernel mapping of User mode 279 * input/output pages. The pages can be accessed through 280 * the mapping without any cache maintenance. 281 * @handle: Handle returned with bind ioctl for creating a 282 * contiguous User mode mapping of input/output pages & 283 * the hardware doorbell page. 284 * @doorbell_nr: Index of the hardware doorbell page assigned to the 285 * queue. 286 * @db_file_offset: File offset value that is assigned to userspace mapping 287 * created on bind to access the doorbell page. 288 * It is in page units. 289 * @link: Link to the linked list of GPU command queues created per 290 * GPU address space. 291 * @refcount: Reference count, stands for the number of times the queue 292 * has been referenced. The reference is taken when it is 293 * created, when it is bound to the group and also when the 294 * @oom_event_work work item is queued 295 * for it. 296 * @group: Pointer to the group to which this queue is bound. 297 * @queue_reg: Pointer to the VA region allocated for CS buffer. 298 * @oom_event_work: Work item corresponding to the out of memory event for 299 * chunked tiler heap being used for this queue. 300 * @base_addr: Base address of the CS buffer. 301 * @size: Size of the CS buffer. 302 * @priority: Priority of this queue within the group. 303 * @bind_state: Bind state of the queue as enum @kbase_csf_queue_bind_state 304 * @csi_index: The ID of the assigned CS hardware interface. 305 * @enabled: Indicating whether the CS is running, or not. 306 * @status_wait: Value of CS_STATUS_WAIT register of the CS will 307 * be kept when the CS gets blocked by sync wait. 308 * CS_STATUS_WAIT provides information on conditions queue is 309 * blocking on. This is set when the group, to which queue is 310 * bound, is suspended after getting blocked, i.e. in 311 * KBASE_CSF_GROUP_SUSPENDED_ON_WAIT_SYNC state. 312 * @sync_ptr: Value of CS_STATUS_WAIT_SYNC_POINTER register of the CS 313 * will be kept when the CS gets blocked by 314 * sync wait. CS_STATUS_WAIT_SYNC_POINTER contains the address 315 * of synchronization object being waited on. 316 * Valid only when @status_wait is set. 317 * @sync_value: Value of CS_STATUS_WAIT_SYNC_VALUE register of the CS 318 * will be kept when the CS gets blocked by 319 * sync wait. CS_STATUS_WAIT_SYNC_VALUE contains the value 320 * tested against the synchronization object. 321 * Valid only when @status_wait is set. 322 * @sb_status: Value indicates which of the scoreboard entries in the queue 323 * are non-zero 324 * @blocked_reason: Value shows if the queue is blocked, and if so, 325 * the reason why it is blocked 326 * @trace_buffer_base: CS trace buffer base address. 327 * @trace_offset_ptr: Pointer to the CS trace buffer offset variable. 328 * @trace_buffer_size: CS trace buffer size for the queue. 329 * @trace_cfg: CS trace configuration parameters. 330 * @error: GPU command queue fatal information to pass to user space. 331 * @fatal_event_work: Work item to handle the CS fatal event reported for this 332 * queue. 333 * @cs_fatal_info: Records additional information about the CS fatal event. 334 * @cs_fatal: Records information about the CS fatal event. 335 * @pending: Indicating whether the queue has new submitted work. 336 */ 337 struct kbase_queue { 338 struct kbase_context *kctx; 339 struct kbase_va_region *reg; 340 struct tagged_addr phys[2]; 341 char *user_io_addr; 342 u64 handle; 343 int doorbell_nr; 344 unsigned long db_file_offset; 345 struct list_head link; 346 atomic_t refcount; 347 struct kbase_queue_group *group; 348 struct kbase_va_region *queue_reg; 349 struct work_struct oom_event_work; 350 u64 base_addr; 351 u32 size; 352 u8 priority; 353 s8 csi_index; 354 enum kbase_csf_queue_bind_state bind_state; 355 bool enabled; 356 u32 status_wait; 357 u64 sync_ptr; 358 u32 sync_value; 359 u32 sb_status; 360 u32 blocked_reason; 361 u64 trace_buffer_base; 362 u64 trace_offset_ptr; 363 u32 trace_buffer_size; 364 u32 trace_cfg; 365 struct kbase_csf_notification error; 366 struct work_struct fatal_event_work; 367 u64 cs_fatal_info; 368 u32 cs_fatal; 369 atomic_t pending; 370 }; 371 372 /** 373 * struct kbase_normal_suspend_buffer - Object representing a normal 374 * suspend buffer for queue group. 375 * @reg: Memory region allocated for the normal-mode suspend buffer. 376 * @phy: Array of physical memory pages allocated for the normal- 377 * mode suspend buffer. 378 */ 379 struct kbase_normal_suspend_buffer { 380 struct kbase_va_region *reg; 381 struct tagged_addr *phy; 382 }; 383 384 /** 385 * struct kbase_protected_suspend_buffer - Object representing a protected 386 * suspend buffer for queue group. 387 * @reg: Memory region allocated for the protected-mode suspend buffer. 388 * @pma: Array of pointer to protected mode allocations containing 389 * information about memory pages allocated for protected mode 390 * suspend buffer. 391 */ 392 struct kbase_protected_suspend_buffer { 393 struct kbase_va_region *reg; 394 struct protected_memory_allocation **pma; 395 }; 396 397 /** 398 * struct kbase_queue_group - Object representing a GPU command queue group. 399 * 400 * @kctx: Pointer to the kbase context with which this queue group 401 * is associated. 402 * @normal_suspend_buf: Object representing the normal suspend buffer. 403 * Normal-mode suspend buffer that is used for 404 * group context switch. 405 * @protected_suspend_buf: Object representing the protected suspend 406 * buffer. Protected-mode suspend buffer that is 407 * used for group context switch. 408 * @handle: Handle which identifies this queue group. 409 * @csg_nr: Number/index of the CSG to which this queue group is 410 * mapped; KBASEP_CSG_NR_INVALID indicates that the queue 411 * group is not scheduled. 412 * @priority: Priority of the queue group, 0 being the highest, 413 * BASE_QUEUE_GROUP_PRIORITY_COUNT - 1 being the lowest. 414 * @tiler_max: Maximum number of tiler endpoints the group is allowed 415 * to use. 416 * @fragment_max: Maximum number of fragment endpoints the group is 417 * allowed to use. 418 * @compute_max: Maximum number of compute endpoints the group is 419 * allowed to use. 420 * @tiler_mask: Mask of tiler endpoints the group is allowed to use. 421 * @fragment_mask: Mask of fragment endpoints the group is allowed to use. 422 * @compute_mask: Mask of compute endpoints the group is allowed to use. 423 * @group_uid: 32-bit wide unsigned identifier for the group, unique 424 * across all kbase devices and contexts. 425 * @link: Link to this queue group in the 'runnable_groups' list of 426 * the corresponding kctx. 427 * @link_to_schedule: Link to this queue group in the list of prepared groups 428 * to be scheduled, if the group is runnable/suspended. 429 * If the group is idle or waiting for CQS, it would be a 430 * link to the list of idle/blocked groups list. 431 * @run_state: Current state of the queue group. 432 * @prepared_seq_num: Indicates the position of queue group in the list of 433 * prepared groups to be scheduled. 434 * @scan_seq_num: Scan out sequence number before adjusting for dynamic 435 * idle conditions. It is used for setting a group's 436 * onslot priority. It could differ from prepared_seq_number 437 * when there are idle groups. 438 * @faulted: Indicates that a GPU fault occurred for the queue group. 439 * This flag persists until the fault has been queued to be 440 * reported to userspace. 441 * @bound_queues: Array of registered queues bound to this queue group. 442 * @doorbell_nr: Index of the hardware doorbell page assigned to the 443 * group. 444 * @protm_event_work: Work item corresponding to the protected mode entry 445 * event for this queue. 446 * @protm_pending_bitmap: Bit array to keep a track of CSs that 447 * have pending protected mode entry requests. 448 * @error_fatal: An error of type BASE_GPU_QUEUE_GROUP_ERROR_FATAL to be 449 * returned to userspace if such an error has occurred. 450 * @error_timeout: An error of type BASE_GPU_QUEUE_GROUP_ERROR_TIMEOUT 451 * to be returned to userspace if such an error has occurred. 452 * @error_tiler_oom: An error of type BASE_GPU_QUEUE_GROUP_ERROR_TILER_HEAP_OOM 453 * to be returned to userspace if such an error has occurred. 454 * @timer_event_work: Work item to handle the progress timeout fatal event 455 * for the group. 456 */ 457 struct kbase_queue_group { 458 struct kbase_context *kctx; 459 struct kbase_normal_suspend_buffer normal_suspend_buf; 460 struct kbase_protected_suspend_buffer protected_suspend_buf; 461 u8 handle; 462 s8 csg_nr; 463 u8 priority; 464 465 u8 tiler_max; 466 u8 fragment_max; 467 u8 compute_max; 468 469 u64 tiler_mask; 470 u64 fragment_mask; 471 u64 compute_mask; 472 473 u32 group_uid; 474 475 struct list_head link; 476 struct list_head link_to_schedule; 477 enum kbase_csf_group_state run_state; 478 u32 prepared_seq_num; 479 u32 scan_seq_num; 480 bool faulted; 481 482 struct kbase_queue *bound_queues[MAX_SUPPORTED_STREAMS_PER_GROUP]; 483 484 int doorbell_nr; 485 struct work_struct protm_event_work; 486 DECLARE_BITMAP(protm_pending_bitmap, MAX_SUPPORTED_STREAMS_PER_GROUP); 487 488 struct kbase_csf_notification error_fatal; 489 struct kbase_csf_notification error_timeout; 490 struct kbase_csf_notification error_tiler_oom; 491 492 struct work_struct timer_event_work; 493 494 }; 495 496 /** 497 * struct kbase_csf_kcpu_queue_context - Object representing the kernel CPU 498 * queues for a GPU address space. 499 * 500 * @lock: Lock preventing concurrent access to @array and the @in_use bitmap. 501 * @array: Array of pointers to kernel CPU command queues. 502 * @in_use: Bitmap which indicates which kernel CPU command queues are in use. 503 * @wq: Dedicated workqueue for processing kernel CPU command queues. 504 * @num_cmds: The number of commands that have been enqueued across 505 * all the KCPU command queues. This could be used as a 506 * timestamp to determine the command's enqueueing time. 507 * @jit_cmds_head: A list of the just-in-time memory commands, both 508 * allocate & free, in submission order, protected 509 * by kbase_csf_kcpu_queue_context.lock. 510 * @jit_blocked_queues: A list of KCPU command queues blocked by a pending 511 * just-in-time memory allocation command which will be 512 * reattempted after the impending free of other active 513 * allocations. 514 */ 515 struct kbase_csf_kcpu_queue_context { 516 struct mutex lock; 517 struct kbase_kcpu_command_queue *array[KBASEP_MAX_KCPU_QUEUES]; 518 DECLARE_BITMAP(in_use, KBASEP_MAX_KCPU_QUEUES); 519 struct workqueue_struct *wq; 520 u64 num_cmds; 521 522 struct list_head jit_cmds_head; 523 struct list_head jit_blocked_queues; 524 }; 525 526 /** 527 * struct kbase_csf_cpu_queue_context - Object representing the cpu queue 528 * information. 529 * 530 * @buffer: Buffer containing CPU queue information provided by Userspace. 531 * @buffer_size: The size of @buffer. 532 * @dump_req_status: Indicates the current status for CPU queues dump request. 533 * @dump_cmp: Dumping cpu queue completion event. 534 */ 535 struct kbase_csf_cpu_queue_context { 536 char *buffer; 537 size_t buffer_size; 538 atomic_t dump_req_status; 539 struct completion dump_cmp; 540 }; 541 542 /** 543 * struct kbase_csf_heap_context_allocator - Allocator of heap contexts 544 * 545 * @kctx: Pointer to the kbase context with which this allocator is 546 * associated. 547 * @region: Pointer to a GPU memory region from which heap context structures 548 * are allocated. NULL if no heap contexts have been allocated. 549 * @gpu_va: GPU virtual address of the start of the region from which heap 550 * context structures are allocated. 0 if no heap contexts have been 551 * allocated. 552 * @lock: Lock preventing concurrent access to the @in_use bitmap. 553 * @in_use: Bitmap that indicates which heap context structures are currently 554 * allocated (in @region). 555 * 556 * Heap context structures are allocated by the kernel for use by the firmware. 557 * The current implementation subdivides a single GPU memory region for use as 558 * a sparse array. 559 */ 560 struct kbase_csf_heap_context_allocator { 561 struct kbase_context *kctx; 562 struct kbase_va_region *region; 563 u64 gpu_va; 564 struct mutex lock; 565 DECLARE_BITMAP(in_use, MAX_TILER_HEAPS); 566 }; 567 568 /** 569 * struct kbase_csf_tiler_heap_context - Object representing the tiler heaps 570 * context for a GPU address space. 571 * 572 * @lock: Lock to prevent the concurrent access to tiler heaps (after the 573 * initialization), a tiler heap can be terminated whilst an OoM 574 * event is being handled for it. 575 * @list: List of tiler heaps. 576 * @ctx_alloc: Allocator for heap context structures. 577 * @nr_of_heaps: Total number of tiler heaps that were added during the 578 * life time of the context. 579 * 580 * This contains all of the CSF state relating to chunked tiler heaps for one 581 * @kbase_context. It is not the same as a heap context structure allocated by 582 * the kernel for use by the firmware. 583 */ 584 struct kbase_csf_tiler_heap_context { 585 struct mutex lock; 586 struct list_head list; 587 struct kbase_csf_heap_context_allocator ctx_alloc; 588 u64 nr_of_heaps; 589 }; 590 591 /** 592 * struct kbase_csf_scheduler_context - Object representing the scheduler's 593 * context for a GPU address space. 594 * 595 * @runnable_groups: Lists of runnable GPU command queue groups in the kctx, 596 * one per queue group relative-priority level. 597 * @num_runnable_grps: Total number of runnable groups across all priority 598 * levels in @runnable_groups. 599 * @idle_wait_groups: A list of GPU command queue groups in which all enabled 600 * GPU command queues are idle and at least one of them 601 * is blocked on a sync wait operation. 602 * @num_idle_wait_grps: Length of the @idle_wait_groups list. 603 * @sync_update_wq: Dedicated workqueue to process work items corresponding 604 * to the sync_update events by sync_set/sync_add 605 * instruction execution on CSs bound to groups 606 * of @idle_wait_groups list. 607 * @sync_update_work: work item to process the sync_update events by 608 * sync_set / sync_add instruction execution on command 609 * streams bound to groups of @idle_wait_groups list. 610 * @ngrp_to_schedule: Number of groups added for the context to the 611 * 'groups_to_schedule' list of scheduler instance. 612 */ 613 struct kbase_csf_scheduler_context { 614 struct list_head runnable_groups[KBASE_QUEUE_GROUP_PRIORITY_COUNT]; 615 u32 num_runnable_grps; 616 struct list_head idle_wait_groups; 617 u32 num_idle_wait_grps; 618 struct workqueue_struct *sync_update_wq; 619 struct work_struct sync_update_work; 620 u32 ngrp_to_schedule; 621 }; 622 623 /** 624 * enum kbase_csf_event_callback_action - return type for CSF event callbacks. 625 * 626 * @KBASE_CSF_EVENT_CALLBACK_FIRST: Never set explicitly. 627 * It doesn't correspond to any action or type of event callback. 628 * 629 * @KBASE_CSF_EVENT_CALLBACK_KEEP: The callback will remain registered. 630 * 631 * @KBASE_CSF_EVENT_CALLBACK_REMOVE: The callback will be removed 632 * immediately upon return. 633 * 634 * @KBASE_CSF_EVENT_CALLBACK_LAST: Never set explicitly. 635 * It doesn't correspond to any action or type of event callback. 636 */ 637 enum kbase_csf_event_callback_action { 638 KBASE_CSF_EVENT_CALLBACK_FIRST = 0, 639 KBASE_CSF_EVENT_CALLBACK_KEEP, 640 KBASE_CSF_EVENT_CALLBACK_REMOVE, 641 KBASE_CSF_EVENT_CALLBACK_LAST, 642 }; 643 644 /** 645 * struct kbase_csf_event - Object representing CSF event and error 646 * 647 * @callback_list: List of callbacks which are registered to serve CSF 648 * events. 649 * @error_list: List for CS fatal errors in CSF context. 650 * Link of fatal error is &struct_kbase_csf_notification.link. 651 * @lock: Lock protecting access to @callback_list and 652 * @error_list. 653 */ 654 struct kbase_csf_event { 655 struct list_head callback_list; 656 struct list_head error_list; 657 spinlock_t lock; 658 }; 659 660 /** 661 * struct kbase_csf_context - Object representing CSF for a GPU address space. 662 * 663 * @event_pages_head: A list of pages allocated for the event memory used by 664 * the synchronization objects. A separate list would help 665 * in the fast lookup, since the list is expected to be short 666 * as one page would provide the memory for up to 1K 667 * synchronization objects. 668 * KBASE_PERMANENTLY_MAPPED_MEM_LIMIT_PAGES is the upper 669 * bound on the size of event memory. 670 * @cookies: Bitmask containing of KBASE_CSF_NUM_USER_IO_PAGES_HANDLE 671 * bits, used for creating the User mode CPU mapping in a 672 * deferred manner of a pair of User mode input/output pages 673 * & a hardware doorbell page. 674 * The pages are allocated when a GPU command queue is 675 * bound to a CSG in kbase_csf_queue_bind. 676 * This helps returning unique handles to Userspace from 677 * kbase_csf_queue_bind and later retrieving the pointer to 678 * queue in the mmap handler. 679 * @user_pages_info: Array containing pointers to queue 680 * structures, used in conjunction with cookies bitmask for 681 * providing a mechansim to create a CPU mapping of 682 * input/output pages & hardware doorbell page. 683 * @lock: Serializes accesses to all members, except for ones that 684 * have their own locks. 685 * @queue_groups: Array of registered GPU command queue groups. 686 * @queue_list: Linked list of GPU command queues not yet deregistered. 687 * Note that queues can persist after deregistration if the 688 * userspace mapping created for them on bind operation 689 * hasn't been removed. 690 * @kcpu_queues: Kernel CPU command queues. 691 * @event: CSF event object. 692 * @tiler_heaps: Chunked tiler memory heaps. 693 * @wq: Dedicated workqueue to process work items corresponding 694 * to the OoM events raised for chunked tiler heaps being 695 * used by GPU command queues, and progress timeout events. 696 * @link: Link to this csf context in the 'runnable_kctxs' list of 697 * the scheduler instance 698 * @user_reg_vma: Pointer to the vma corresponding to the virtual mapping 699 * of the USER register page. Currently used only for sanity 700 * checking. 701 * @sched: Object representing the scheduler's context 702 * @pending_submission_work: Work item to process pending kicked GPU command queues. 703 * @cpu_queue: CPU queue information. Only be available when DEBUG_FS 704 * is enabled. 705 */ 706 struct kbase_csf_context { 707 struct list_head event_pages_head; 708 DECLARE_BITMAP(cookies, KBASE_CSF_NUM_USER_IO_PAGES_HANDLE); 709 struct kbase_queue *user_pages_info[ 710 KBASE_CSF_NUM_USER_IO_PAGES_HANDLE]; 711 struct mutex lock; 712 struct kbase_queue_group *queue_groups[MAX_QUEUE_GROUP_NUM]; 713 struct list_head queue_list; 714 struct kbase_csf_kcpu_queue_context kcpu_queues; 715 struct kbase_csf_event event; 716 struct kbase_csf_tiler_heap_context tiler_heaps; 717 struct workqueue_struct *wq; 718 struct list_head link; 719 struct vm_area_struct *user_reg_vma; 720 struct kbase_csf_scheduler_context sched; 721 struct work_struct pending_submission_work; 722 #if IS_ENABLED(CONFIG_DEBUG_FS) 723 struct kbase_csf_cpu_queue_context cpu_queue; 724 #endif 725 }; 726 727 /** 728 * struct kbase_csf_reset_gpu - Object containing the members required for 729 * GPU reset handling. 730 * @workq: Workqueue to execute the GPU reset work item @work. 731 * @work: Work item for performing the GPU reset. 732 * @wait: Wait queue used to wait for the GPU reset completion. 733 * @sem: RW Semaphore to ensure no other thread attempts to use the 734 * GPU whilst a reset is in process. Unlike traditional 735 * semaphores and wait queues, this allows Linux's lockdep 736 * mechanism to check for deadlocks involving reset waits. 737 * @state: Tracks if the GPU reset is in progress or not. 738 * The state is represented by enum @kbase_csf_reset_gpu_state. 739 */ 740 struct kbase_csf_reset_gpu { 741 struct workqueue_struct *workq; 742 struct work_struct work; 743 wait_queue_head_t wait; 744 struct rw_semaphore sem; 745 atomic_t state; 746 }; 747 748 /** 749 * struct kbase_csf_csg_slot - Object containing members for tracking the state 750 * of CSG slots. 751 * @resident_group: pointer to the queue group that is resident on the CSG slot. 752 * @state: state of the slot as per enum @kbase_csf_csg_slot_state. 753 * @trigger_jiffies: value of jiffies when change in slot state is recorded. 754 * @priority: dynamic priority assigned to CSG slot. 755 */ 756 struct kbase_csf_csg_slot { 757 struct kbase_queue_group *resident_group; 758 atomic_t state; 759 unsigned long trigger_jiffies; 760 u8 priority; 761 }; 762 763 /** 764 * struct kbase_csf_scheduler - Object representing the scheduler used for 765 * CSF for an instance of GPU platform device. 766 * @lock: Lock to serialize the scheduler operations and 767 * access to the data members. 768 * @interrupt_lock: Lock to protect members accessed by interrupt 769 * handler. 770 * @state: The operational phase the scheduler is in. Primarily 771 * used for indicating what in-cycle schedule actions 772 * are allowed. 773 * @doorbell_inuse_bitmap: Bitmap of hardware doorbell pages keeping track of 774 * which pages are currently available for assignment 775 * to clients. 776 * @csg_inuse_bitmap: Bitmap to keep a track of CSG slots 777 * that are currently in use. 778 * @csg_slots: The array for tracking the state of CS 779 * group slots. 780 * @runnable_kctxs: List of Kbase contexts that have runnable command 781 * queue groups. 782 * @groups_to_schedule: List of runnable queue groups prepared on every 783 * scheduler tick. The dynamic priority of the CSG 784 * slot assigned to a group will depend upon the 785 * position of group in the list. 786 * @ngrp_to_schedule: Number of groups in the @groups_to_schedule list, 787 * incremented when a group is added to the list, used 788 * to record the position of group in the list. 789 * @num_active_address_spaces: Number of GPU address space slots that would get 790 * used to program the groups in @groups_to_schedule 791 * list on all the available CSG 792 * slots. 793 * @num_csg_slots_for_tick: Number of CSG slots that can be 794 * active in the given tick/tock. This depends on the 795 * value of @num_active_address_spaces. 796 * @remaining_tick_slots: Tracking the number of remaining available slots 797 * for @num_csg_slots_for_tick during the scheduling 798 * operation in a tick/tock. 799 * @idle_groups_to_schedule: List of runnable queue groups, in which all GPU 800 * command queues became idle or are waiting for 801 * synchronization object, prepared on every 802 * scheduler tick. The groups in this list are 803 * appended to the tail of @groups_to_schedule list 804 * after the scan out so that the idle groups aren't 805 * preferred for scheduling over the non-idle ones. 806 * @csg_scan_count_for_tick: CSG scanout count for assign the scan_seq_num for 807 * each scanned out group during scheduling operation 808 * in a tick/tock. 809 * @total_runnable_grps: Total number of runnable groups across all KCTXs. 810 * @csgs_events_enable_mask: Use for temporary masking off asynchronous events 811 * from firmware (such as OoM events) before a group 812 * is suspended. 813 * @csg_slots_idle_mask: Bit array for storing the mask of CS 814 * group slots for which idle notification was 815 * received. 816 * @csg_slots_prio_update: Bit array for tracking slots that have an on-slot 817 * priority update operation. 818 * @last_schedule: Time in jiffies recorded when the last "tick" or 819 * "tock" schedule operation concluded. Used for 820 * evaluating the exclusion window for in-cycle 821 * schedule operation. 822 * @timer_enabled: Whether the CSF scheduler wakes itself up for 823 * periodic scheduling tasks. If this value is 0 824 * then it will only perform scheduling under the 825 * influence of external factors e.g., IRQs, IOCTLs. 826 * @wq: Dedicated workqueue to execute the @tick_work. 827 * @tick_timer: High-resolution timer employed to schedule tick 828 * workqueue items (kernel-provided delayed_work 829 * items do not use hrtimer and for some reason do 830 * not provide sufficiently reliable periodicity). 831 * @tick_work: Work item that performs the "schedule on tick" 832 * operation to implement timeslice-based scheduling. 833 * @tock_work: Work item that would perform the schedule on tock 834 * operation to implement the asynchronous scheduling. 835 * @ping_work: Work item that would ping the firmware at regular 836 * intervals, only if there is a single active CSG 837 * slot, to check if firmware is alive and would 838 * initiate a reset if the ping request isn't 839 * acknowledged. 840 * @top_ctx: Pointer to the Kbase context corresponding to the 841 * @top_grp. 842 * @top_grp: Pointer to queue group inside @groups_to_schedule 843 * list that was assigned the highest slot priority. 844 * @tock_pending_request: A "tock" request is pending: a group that is not 845 * currently on the GPU demands to be scheduled. 846 * @active_protm_grp: Indicates if firmware has been permitted to let GPU 847 * enter protected mode with the given group. On exit 848 * from protected mode the pointer is reset to NULL. 849 * This pointer is set and PROTM_ENTER request is sent 850 * atomically with @interrupt_lock held. 851 * This pointer being set doesn't necessarily indicates 852 * that GPU is in protected mode, kbdev->protected_mode 853 * needs to be checked for that. 854 * @gpu_idle_fw_timer_enabled: Whether the CSF scheduler has activiated the 855 * firmware idle hysteresis timer for preparing a 856 * GPU suspend on idle. 857 * @gpu_idle_work: Work item for facilitating the scheduler to bring 858 * the GPU to a low-power mode on becoming idle. 859 * @non_idle_offslot_grps: Count of off-slot non-idle groups. Reset during 860 * the scheduler active phase in a tick. It then 861 * tracks the count of non-idle groups across all the 862 * other phases. 863 * @non_idle_scanout_grps: Count on the non-idle groups in the scan-out 864 * list at the scheduling prepare stage. 865 * @pm_active_count: Count indicating if the scheduler is owning a power 866 * management reference count. Reference is taken when 867 * the count becomes 1 and is dropped when the count 868 * becomes 0. It is used to enable the power up of MCU 869 * after GPU and L2 cache have been powered up. So when 870 * this count is zero, MCU will not be powered up. 871 * @csg_scheduling_period_ms: Duration of Scheduling tick in milliseconds. 872 * @tick_timer_active: Indicates whether the @tick_timer is effectively 873 * active or not, as the callback function of 874 * @tick_timer will enqueue @tick_work only if this 875 * flag is true. This is mainly useful for the case 876 * when scheduling tick needs to be advanced from 877 * interrupt context, without actually deactivating 878 * the @tick_timer first and then enqueing @tick_work. 879 */ 880 struct kbase_csf_scheduler { 881 struct mutex lock; 882 spinlock_t interrupt_lock; 883 enum kbase_csf_scheduler_state state; 884 DECLARE_BITMAP(doorbell_inuse_bitmap, CSF_NUM_DOORBELL); 885 DECLARE_BITMAP(csg_inuse_bitmap, MAX_SUPPORTED_CSGS); 886 struct kbase_csf_csg_slot *csg_slots; 887 struct list_head runnable_kctxs; 888 struct list_head groups_to_schedule; 889 u32 ngrp_to_schedule; 890 u32 num_active_address_spaces; 891 u32 num_csg_slots_for_tick; 892 u32 remaining_tick_slots; 893 struct list_head idle_groups_to_schedule; 894 u32 csg_scan_count_for_tick; 895 u32 total_runnable_grps; 896 DECLARE_BITMAP(csgs_events_enable_mask, MAX_SUPPORTED_CSGS); 897 DECLARE_BITMAP(csg_slots_idle_mask, MAX_SUPPORTED_CSGS); 898 DECLARE_BITMAP(csg_slots_prio_update, MAX_SUPPORTED_CSGS); 899 unsigned long last_schedule; 900 bool timer_enabled; 901 struct workqueue_struct *wq; 902 struct hrtimer tick_timer; 903 struct work_struct tick_work; 904 struct delayed_work tock_work; 905 struct delayed_work ping_work; 906 struct kbase_context *top_ctx; 907 struct kbase_queue_group *top_grp; 908 bool tock_pending_request; 909 struct kbase_queue_group *active_protm_grp; 910 bool gpu_idle_fw_timer_enabled; 911 struct work_struct gpu_idle_work; 912 atomic_t non_idle_offslot_grps; 913 u32 non_idle_scanout_grps; 914 u32 pm_active_count; 915 unsigned int csg_scheduling_period_ms; 916 bool tick_timer_active; 917 }; 918 919 /* 920 * Number of GPU cycles per unit of the global progress timeout. 921 */ 922 #define GLB_PROGRESS_TIMER_TIMEOUT_SCALE ((u64)1024) 923 924 /* 925 * Maximum value of the global progress timeout. 926 */ 927 #define GLB_PROGRESS_TIMER_TIMEOUT_MAX \ 928 ((GLB_PROGRESS_TIMER_TIMEOUT_MASK >> \ 929 GLB_PROGRESS_TIMER_TIMEOUT_SHIFT) * \ 930 GLB_PROGRESS_TIMER_TIMEOUT_SCALE) 931 932 /* 933 * Default GLB_PWROFF_TIMER_TIMEOUT value in unit of micro-seconds. 934 */ 935 #define DEFAULT_GLB_PWROFF_TIMEOUT_US (800) 936 937 /* 938 * In typical operations, the management of the shader core power transitions 939 * is delegated to the MCU/firmware. However, if the host driver is configured 940 * to take direct control, one needs to disable the MCU firmware GLB_PWROFF 941 * timer. 942 */ 943 #define DISABLE_GLB_PWROFF_TIMER (0) 944 945 /* Index of the GPU_ACTIVE counter within the CSHW counter block */ 946 #define GPU_ACTIVE_CNT_IDX (4) 947 948 /* 949 * Maximum number of sessions that can be managed by the IPA Control component. 950 */ 951 #if MALI_UNIT_TEST 952 #define KBASE_IPA_CONTROL_MAX_SESSIONS ((size_t)8) 953 #else 954 #define KBASE_IPA_CONTROL_MAX_SESSIONS ((size_t)2) 955 #endif 956 957 /** 958 * enum kbase_ipa_core_type - Type of counter block for performance counters 959 * 960 * @KBASE_IPA_CORE_TYPE_CSHW: CS Hardware counters. 961 * @KBASE_IPA_CORE_TYPE_MEMSYS: Memory System counters. 962 * @KBASE_IPA_CORE_TYPE_TILER: Tiler counters. 963 * @KBASE_IPA_CORE_TYPE_SHADER: Shader Core counters. 964 * @KBASE_IPA_CORE_TYPE_NUM: Number of core types. 965 */ 966 enum kbase_ipa_core_type { 967 KBASE_IPA_CORE_TYPE_CSHW = 0, 968 KBASE_IPA_CORE_TYPE_MEMSYS, 969 KBASE_IPA_CORE_TYPE_TILER, 970 KBASE_IPA_CORE_TYPE_SHADER, 971 KBASE_IPA_CORE_TYPE_NUM 972 }; 973 974 /* 975 * Number of configurable counters per type of block on the IPA Control 976 * interface. 977 */ 978 #define KBASE_IPA_CONTROL_NUM_BLOCK_COUNTERS ((size_t)8) 979 980 /* 981 * Total number of configurable counters existing on the IPA Control interface. 982 */ 983 #define KBASE_IPA_CONTROL_MAX_COUNTERS \ 984 ((size_t)KBASE_IPA_CORE_TYPE_NUM * KBASE_IPA_CONTROL_NUM_BLOCK_COUNTERS) 985 986 /** 987 * struct kbase_ipa_control_prfcnt - Session for a single performance counter 988 * 989 * @latest_raw_value: Latest raw value read from the counter. 990 * @scaling_factor: Factor raw value shall be multiplied by. 991 * @accumulated_diff: Partial sum of scaled and normalized values from 992 * previous samples. This represent all the values 993 * that were read before the latest raw value. 994 * @type: Type of counter block for performance counter. 995 * @select_idx: Index of the performance counter as configured on 996 * the IPA Control interface. 997 * @gpu_norm: Indicating whether values shall be normalized by 998 * GPU frequency. If true, returned values represent 999 * an interval of time expressed in seconds (when the 1000 * scaling factor is set to 1). 1001 */ 1002 struct kbase_ipa_control_prfcnt { 1003 u64 latest_raw_value; 1004 u64 scaling_factor; 1005 u64 accumulated_diff; 1006 enum kbase_ipa_core_type type; 1007 u8 select_idx; 1008 bool gpu_norm; 1009 }; 1010 1011 /** 1012 * struct kbase_ipa_control_session - Session for an IPA Control client 1013 * 1014 * @prfcnts: Sessions for individual performance counters. 1015 * @num_prfcnts: Number of performance counters. 1016 * @active: Indicates whether this slot is in use or not 1017 * @last_query_time: Time of last query, in ns 1018 * @protm_time: Amount of time (in ns) that GPU has been in protected 1019 */ 1020 struct kbase_ipa_control_session { 1021 struct kbase_ipa_control_prfcnt prfcnts[KBASE_IPA_CONTROL_MAX_COUNTERS]; 1022 size_t num_prfcnts; 1023 bool active; 1024 u64 last_query_time; 1025 u64 protm_time; 1026 }; 1027 1028 /** 1029 * struct kbase_ipa_control_prfcnt_config - Performance counter configuration 1030 * 1031 * @idx: Index of the performance counter inside the block, as specified 1032 * in the GPU architecture. 1033 * @refcount: Number of client sessions bound to this counter. 1034 * 1035 * This structure represents one configurable performance counter of 1036 * the IPA Control interface. The entry may be mapped to a specific counter 1037 * by one or more client sessions. The counter is considered to be unused 1038 * if it isn't part of any client session. 1039 */ 1040 struct kbase_ipa_control_prfcnt_config { 1041 u8 idx; 1042 u8 refcount; 1043 }; 1044 1045 /** 1046 * struct kbase_ipa_control_prfcnt_block - Block of performance counters 1047 * 1048 * @select: Current performance counter configuration. 1049 * @num_available_counters: Number of counters that are not already configured. 1050 * 1051 */ 1052 struct kbase_ipa_control_prfcnt_block { 1053 struct kbase_ipa_control_prfcnt_config 1054 select[KBASE_IPA_CONTROL_NUM_BLOCK_COUNTERS]; 1055 size_t num_available_counters; 1056 }; 1057 1058 /** 1059 * struct kbase_ipa_control - Manager of the IPA Control interface. 1060 * 1061 * @blocks: Current configuration of performance counters 1062 * for the IPA Control interface. 1063 * @sessions: State of client sessions, storing information 1064 * like performance counters the client subscribed to 1065 * and latest value read from each counter. 1066 * @lock: Spinlock to serialize access by concurrent clients. 1067 * @rtm_listener_data: Private data for allocating a GPU frequency change 1068 * listener. 1069 * @num_active_sessions: Number of sessions opened by clients. 1070 * @cur_gpu_rate: Current GPU top-level operating frequency, in Hz. 1071 * @rtm_listener_data: Private data for allocating a GPU frequency change 1072 * listener. 1073 * @protm_start: Time (in ns) at which the GPU entered protected mode 1074 */ 1075 struct kbase_ipa_control { 1076 struct kbase_ipa_control_prfcnt_block blocks[KBASE_IPA_CORE_TYPE_NUM]; 1077 struct kbase_ipa_control_session 1078 sessions[KBASE_IPA_CONTROL_MAX_SESSIONS]; 1079 spinlock_t lock; 1080 void *rtm_listener_data; 1081 size_t num_active_sessions; 1082 u32 cur_gpu_rate; 1083 u64 protm_start; 1084 }; 1085 1086 /** 1087 * struct kbase_csf_firmware_interface - Interface in the MCU firmware 1088 * 1089 * @node: Interface objects are on the kbase_device:csf.firmware_interfaces 1090 * list using this list_head to link them 1091 * @phys: Array of the physical (tagged) addresses making up this interface 1092 * @name: NULL-terminated string naming the interface 1093 * @num_pages: Number of entries in @phys and @pma (and length of the interface) 1094 * @virtual: Starting GPU virtual address this interface is mapped at 1095 * @flags: bitmask of CSF_FIRMWARE_ENTRY_* conveying the interface attributes 1096 * @data_start: Offset into firmware image at which the interface data starts 1097 * @data_end: Offset into firmware image at which the interface data ends 1098 * @kernel_map: A kernel mapping of the memory or NULL if not required to be 1099 * mapped in the kernel 1100 * @pma: Array of pointers to protected memory allocations. 1101 */ 1102 struct kbase_csf_firmware_interface { 1103 struct list_head node; 1104 struct tagged_addr *phys; 1105 char *name; 1106 u32 num_pages; 1107 u32 virtual; 1108 u32 flags; 1109 u32 data_start; 1110 u32 data_end; 1111 void *kernel_map; 1112 struct protected_memory_allocation **pma; 1113 }; 1114 1115 /* 1116 * struct kbase_csf_hwcnt - Object containing members for handling the dump of 1117 * HW counters. 1118 * 1119 * @request_pending: Flag set when HWC requested and used for HWC sample 1120 * done interrupt. 1121 * @enable_pending: Flag set when HWC enable status change and used for 1122 * enable done interrupt. 1123 */ 1124 struct kbase_csf_hwcnt { 1125 bool request_pending; 1126 bool enable_pending; 1127 }; 1128 1129 /** 1130 * struct kbase_csf_device - Object representing CSF for an instance of GPU 1131 * platform device. 1132 * 1133 * @mcu_mmu: MMU page tables for the MCU firmware 1134 * @firmware_interfaces: List of interfaces defined in the firmware image 1135 * @firmware_config: List of configuration options within the firmware 1136 * image 1137 * @firmware_timeline_metadata: List of timeline meta-data within the firmware 1138 * image 1139 * @fw_cfg_kobj: Pointer to the kobject corresponding to the sysf 1140 * directory that contains a sub-directory for each 1141 * of the configuration option present in the 1142 * firmware image. 1143 * @firmware_trace_buffers: List of trace buffers described in the firmware 1144 * image. 1145 * @shared_interface: Pointer to the interface object containing info for 1146 * the memory area shared between firmware & host. 1147 * @shared_reg_rbtree: RB tree of the memory regions allocated from the 1148 * shared interface segment in MCU firmware address 1149 * space. 1150 * @db_filp: Pointer to a dummy file, that alongwith 1151 * @db_file_offsets, facilitates the use of unqiue 1152 * file offset for the userspace mapping created 1153 * for Hw Doorbell pages. The userspace mapping 1154 * is made to point to this file inside the mmap 1155 * handler. 1156 * @db_file_offsets: Counter that is incremented every time a GPU 1157 * command queue is bound to provide a unique file 1158 * offset range for @db_filp file, so that pte of 1159 * Doorbell page can be zapped through the kernel 1160 * function unmap_mapping_range(). It is incremented 1161 * in page units. 1162 * @dummy_db_page: Address of the dummy page that is mapped in place 1163 * of the real Hw doorbell page for the active GPU 1164 * command queues after they are stopped or after the 1165 * GPU is powered down. 1166 * @dummy_user_reg_page: Address of the dummy page that is mapped in place 1167 * of the real User register page just before the GPU 1168 * is powered down. The User register page is mapped 1169 * in the address space of every process, that created 1170 * a Base context, to enable the access to LATEST_FLUSH 1171 * register from userspace. 1172 * @mali_file_inode: Pointer to the inode corresponding to mali device 1173 * file. This is needed in order to switch to the 1174 * @dummy_user_reg_page on GPU power down. 1175 * All instances of the mali device file will point to 1176 * the same inode. 1177 * @reg_lock: Lock to serialize the MCU firmware related actions 1178 * that affect all contexts such as allocation of 1179 * regions from shared interface area, assignment of 1180 * of hardware doorbell pages, assignment of CSGs, 1181 * sending global requests. 1182 * @event_wait: Wait queue to wait for receiving csf events, i.e. 1183 * the interrupt from CSF firmware, or scheduler state 1184 * changes. 1185 * @interrupt_received: Flag set when the interrupt is received from CSF fw 1186 * @global_iface: The result of parsing the global interface 1187 * structure set up by the firmware, including the 1188 * CSGs, CSs, and their properties 1189 * @scheduler: The CS scheduler instance. 1190 * @reset: Contain members required for GPU reset handling. 1191 * @progress_timeout: Maximum number of GPU clock cycles without forward 1192 * progress to allow, for all tasks running on 1193 * hardware endpoints (e.g. shader cores), before 1194 * terminating a GPU command queue group. 1195 * Must not exceed @GLB_PROGRESS_TIMER_TIMEOUT_MAX. 1196 * @pma_dev: Pointer to protected memory allocator device. 1197 * @firmware_inited: Flag for indicating that the cold-boot stage of 1198 * the MCU has completed. 1199 * @firmware_reloaded: Flag for indicating a firmware reload operation 1200 * in GPU reset has completed. 1201 * @firmware_reload_needed: Flag for indicating that the firmware needs to be 1202 * reloaded as part of the GPU reset action. 1203 * @firmware_hctl_core_pwr: Flag for indicating that the host diver is in 1204 * charge of the shader core's power transitions, and 1205 * the mcu_core_pwroff timeout feature is disabled 1206 * (i.e. configured 0 in the register field). If 1207 * false, the control is delegated to the MCU. 1208 * @firmware_reload_work: Work item for facilitating the procedural actions 1209 * on reloading the firmware. 1210 * @glb_init_request_pending: Flag to indicate that Global requests have been 1211 * sent to the FW after MCU was re-enabled and their 1212 * acknowledgement is pending. 1213 * @fw_error_work: Work item for handling the firmware internal error 1214 * fatal event. 1215 * @ipa_control: IPA Control component manager. 1216 * @mcu_core_pwroff_dur_us: Sysfs attribute for the glb_pwroff timeout input 1217 * in unit of micro-seconds. The firmware does not use 1218 * it directly. 1219 * @mcu_core_pwroff_dur_count: The counterpart of the glb_pwroff timeout input 1220 * in interface required format, ready to be used 1221 * directly in the firmware. 1222 * @mcu_core_pwroff_reg_shadow: The actual value that has been programed into 1223 * the glb_pwoff register. This is separated from 1224 * the @p mcu_core_pwroff_dur_count as an update 1225 * to the latter is asynchronous. 1226 * @gpu_idle_hysteresis_ms: Sysfs attribute for the idle hysteresis time 1227 * window in unit of ms. The firmware does not use it 1228 * directly. 1229 * @gpu_idle_dur_count: The counterpart of the hysteresis time window in 1230 * interface required format, ready to be used 1231 * directly in the firmware. 1232 * @fw_timeout_ms: Timeout value (in milliseconds) used when waiting 1233 * for any request sent to the firmware. 1234 * @hwcnt: Contain members required for handling the dump of 1235 * HW counters. 1236 */ 1237 struct kbase_csf_device { 1238 struct kbase_mmu_table mcu_mmu; 1239 struct list_head firmware_interfaces; 1240 struct list_head firmware_config; 1241 struct list_head firmware_timeline_metadata; 1242 struct kobject *fw_cfg_kobj; 1243 struct kbase_csf_trace_buffers firmware_trace_buffers; 1244 void *shared_interface; 1245 struct rb_root shared_reg_rbtree; 1246 struct file *db_filp; 1247 u32 db_file_offsets; 1248 struct tagged_addr dummy_db_page; 1249 struct tagged_addr dummy_user_reg_page; 1250 struct inode *mali_file_inode; 1251 struct mutex reg_lock; 1252 wait_queue_head_t event_wait; 1253 bool interrupt_received; 1254 struct kbase_csf_global_iface global_iface; 1255 struct kbase_csf_scheduler scheduler; 1256 struct kbase_csf_reset_gpu reset; 1257 atomic64_t progress_timeout; 1258 struct protected_memory_allocator_device *pma_dev; 1259 bool firmware_inited; 1260 bool firmware_reloaded; 1261 bool firmware_reload_needed; 1262 bool firmware_hctl_core_pwr; 1263 struct work_struct firmware_reload_work; 1264 bool glb_init_request_pending; 1265 struct work_struct fw_error_work; 1266 struct kbase_ipa_control ipa_control; 1267 u32 mcu_core_pwroff_dur_us; 1268 u32 mcu_core_pwroff_dur_count; 1269 u32 mcu_core_pwroff_reg_shadow; 1270 u32 gpu_idle_hysteresis_ms; 1271 u32 gpu_idle_dur_count; 1272 unsigned int fw_timeout_ms; 1273 struct kbase_csf_hwcnt hwcnt; 1274 }; 1275 1276 /** 1277 * struct kbase_as - Object representing an address space of GPU. 1278 * @number: Index at which this address space structure is present 1279 * in an array of address space structures embedded inside 1280 * the &struct kbase_device. 1281 * @pf_wq: Workqueue for processing work items related to 1282 * Page fault, Bus fault and GPU fault handling. 1283 * @work_pagefault: Work item for the Page fault handling. 1284 * @work_busfault: Work item for the Bus fault handling. 1285 * @work_gpufault: Work item for the GPU fault handling. 1286 * @pf_data: Data relating to Page fault. 1287 * @bf_data: Data relating to Bus fault. 1288 * @gf_data: Data relating to GPU fault. 1289 * @current_setup: Stores the MMU configuration for this address space. 1290 */ 1291 struct kbase_as { 1292 int number; 1293 struct workqueue_struct *pf_wq; 1294 struct work_struct work_pagefault; 1295 struct work_struct work_busfault; 1296 struct work_struct work_gpufault; 1297 struct kbase_fault pf_data; 1298 struct kbase_fault bf_data; 1299 struct kbase_fault gf_data; 1300 struct kbase_mmu_setup current_setup; 1301 }; 1302 1303 #endif /* _KBASE_CSF_DEFS_H_ */ 1304