1 /* SPDX-License-Identifier: GPL-2.0 2 * 3 * Copyright 2019-2022 HabanaLabs, Ltd. 4 * All Rights Reserved. 5 * 6 */ 7 8 #ifndef GAUDIP_H_ 9 #define GAUDIP_H_ 10 11 #include <uapi/misc/habanalabs.h> 12 #include "../common/habanalabs.h" 13 #include "../include/common/hl_boot_if.h" 14 #include "../include/gaudi/gaudi_packets.h" 15 #include "../include/gaudi/gaudi.h" 16 #include "../include/gaudi/gaudi_async_events.h" 17 #include "../include/gaudi/gaudi_fw_if.h" 18 19 #define NUMBER_OF_EXT_HW_QUEUES 8 20 #define NUMBER_OF_CMPLT_QUEUES NUMBER_OF_EXT_HW_QUEUES 21 #define NUMBER_OF_CPU_HW_QUEUES 1 22 #define NUMBER_OF_INT_HW_QUEUES 100 23 #define NUMBER_OF_HW_QUEUES (NUMBER_OF_EXT_HW_QUEUES + \ 24 NUMBER_OF_CPU_HW_QUEUES + \ 25 NUMBER_OF_INT_HW_QUEUES) 26 27 /* 10 NIC QMANs, DMA5 QMAN, TPC7 QMAN */ 28 #define NUMBER_OF_COLLECTIVE_QUEUES 12 29 #define NUMBER_OF_SOBS_IN_GRP 11 30 31 /* 32 * Number of MSI interrupts IDS: 33 * Each completion queue has 1 ID 34 * The event queue has 1 ID 35 */ 36 #define NUMBER_OF_INTERRUPTS (NUMBER_OF_CMPLT_QUEUES + \ 37 NUMBER_OF_CPU_HW_QUEUES) 38 39 #define GAUDI_STREAM_MASTER_ARR_SIZE 8 40 41 #if (NUMBER_OF_INTERRUPTS > GAUDI_MSI_ENTRIES) 42 #error "Number of MSI interrupts must be smaller or equal to GAUDI_MSI_ENTRIES" 43 #endif 44 45 #define CORESIGHT_TIMEOUT_USEC 100000 /* 100 ms */ 46 47 #define GAUDI_MAX_CLK_FREQ 2200000000ull /* 2200 MHz */ 48 49 #define MAX_POWER_DEFAULT_PCI 200000 /* 200W */ 50 #define MAX_POWER_DEFAULT_PMC 350000 /* 350W */ 51 52 #define DC_POWER_DEFAULT_PCI 60000 /* 60W */ 53 #define DC_POWER_DEFAULT_PMC 60000 /* 60W */ 54 55 #define DC_POWER_DEFAULT_PMC_SEC 97000 /* 97W */ 56 57 #define GAUDI_CPU_TIMEOUT_USEC 30000000 /* 30s */ 58 59 #define TPC_ENABLED_MASK 0xFF 60 61 #define GAUDI_HBM_SIZE_32GB 0x800000000ull 62 #define GAUDI_HBM_DEVICES 4 63 #define GAUDI_HBM_CHANNELS 8 64 #define GAUDI_HBM_CFG_BASE (mmHBM0_BASE - CFG_BASE) 65 #define GAUDI_HBM_CFG_OFFSET (mmHBM1_BASE - mmHBM0_BASE) 66 67 #define DMA_MAX_TRANSFER_SIZE U32_MAX 68 69 #define GAUDI_DEFAULT_CARD_NAME "HL205" 70 71 #define GAUDI_MAX_PENDING_CS SZ_16K 72 73 #if !IS_MAX_PENDING_CS_VALID(GAUDI_MAX_PENDING_CS) 74 #error "GAUDI_MAX_PENDING_CS must be power of 2 and greater than 1" 75 #endif 76 77 #define PCI_DMA_NUMBER_OF_CHNLS 2 78 #define HBM_DMA_NUMBER_OF_CHNLS 6 79 #define DMA_NUMBER_OF_CHNLS (PCI_DMA_NUMBER_OF_CHNLS + \ 80 HBM_DMA_NUMBER_OF_CHNLS) 81 82 #define MME_NUMBER_OF_SLAVE_ENGINES 2 83 #define MME_NUMBER_OF_ENGINES (MME_NUMBER_OF_MASTER_ENGINES + \ 84 MME_NUMBER_OF_SLAVE_ENGINES) 85 #define MME_NUMBER_OF_QMANS (MME_NUMBER_OF_MASTER_ENGINES * \ 86 QMAN_STREAMS) 87 88 #define QMAN_STREAMS 4 89 #define PQ_FETCHER_CACHE_SIZE 8 90 91 #define DMA_QMAN_OFFSET (mmDMA1_QM_BASE - mmDMA0_QM_BASE) 92 #define TPC_QMAN_OFFSET (mmTPC1_QM_BASE - mmTPC0_QM_BASE) 93 #define MME_QMAN_OFFSET (mmMME1_QM_BASE - mmMME0_QM_BASE) 94 #define NIC_MACRO_QMAN_OFFSET (mmNIC1_QM0_BASE - mmNIC0_QM0_BASE) 95 #define NIC_ENGINE_QMAN_OFFSET (mmNIC0_QM1_BASE - mmNIC0_QM0_BASE) 96 97 #define TPC_CFG_OFFSET (mmTPC1_CFG_BASE - mmTPC0_CFG_BASE) 98 99 #define DMA_CORE_OFFSET (mmDMA1_CORE_BASE - mmDMA0_CORE_BASE) 100 101 #define QMAN_LDMA_SRC_OFFSET (mmDMA0_CORE_SRC_BASE_LO - mmDMA0_CORE_CFG_0) 102 #define QMAN_LDMA_DST_OFFSET (mmDMA0_CORE_DST_BASE_LO - mmDMA0_CORE_CFG_0) 103 #define QMAN_LDMA_SIZE_OFFSET (mmDMA0_CORE_DST_TSIZE_0 - mmDMA0_CORE_CFG_0) 104 105 #define QMAN_CPDMA_SRC_OFFSET (mmDMA0_QM_CQ_PTR_LO_4 - mmDMA0_CORE_CFG_0) 106 #define QMAN_CPDMA_DST_OFFSET (mmDMA0_CORE_DST_BASE_LO - mmDMA0_CORE_CFG_0) 107 #define QMAN_CPDMA_SIZE_OFFSET (mmDMA0_QM_CQ_TSIZE_4 - mmDMA0_CORE_CFG_0) 108 109 #define SIF_RTR_CTRL_OFFSET (mmSIF_RTR_CTRL_1_BASE - mmSIF_RTR_CTRL_0_BASE) 110 111 #define NIF_RTR_CTRL_OFFSET (mmNIF_RTR_CTRL_1_BASE - mmNIF_RTR_CTRL_0_BASE) 112 113 #define MME_ACC_OFFSET (mmMME1_ACC_BASE - mmMME0_ACC_BASE) 114 #define SRAM_BANK_OFFSET (mmSRAM_Y0_X1_RTR_BASE - mmSRAM_Y0_X0_RTR_BASE) 115 116 #define NUM_OF_SOB_IN_BLOCK \ 117 (((mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_2047 - \ 118 mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0) + 4) >> 2) 119 120 #define NUM_OF_MONITORS_IN_BLOCK \ 121 (((mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_STATUS_511 - \ 122 mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_STATUS_0) + 4) >> 2) 123 124 #define MONITOR_MAX_SOBS 8 125 126 /* DRAM Memory Map */ 127 128 #define CPU_FW_IMAGE_SIZE 0x10000000 /* 256MB */ 129 #define MMU_PAGE_TABLES_SIZE 0x0BF00000 /* 191MB */ 130 #define MMU_CACHE_MNG_SIZE 0x00100000 /* 1MB */ 131 #define RESERVED 0x04000000 /* 64MB */ 132 133 #define CPU_FW_IMAGE_ADDR DRAM_PHYS_BASE 134 #define MMU_PAGE_TABLES_ADDR (CPU_FW_IMAGE_ADDR + CPU_FW_IMAGE_SIZE) 135 #define MMU_CACHE_MNG_ADDR (MMU_PAGE_TABLES_ADDR + MMU_PAGE_TABLES_SIZE) 136 137 #define DRAM_DRIVER_END_ADDR (MMU_CACHE_MNG_ADDR + MMU_CACHE_MNG_SIZE +\ 138 RESERVED) 139 140 #define DRAM_BASE_ADDR_USER 0x20000000 141 142 #if (DRAM_DRIVER_END_ADDR > DRAM_BASE_ADDR_USER) 143 #error "Driver must reserve no more than 512MB" 144 #endif 145 146 /* Internal QMANs PQ sizes */ 147 148 #define MME_QMAN_LENGTH 1024 149 #define MME_QMAN_SIZE_IN_BYTES (MME_QMAN_LENGTH * QMAN_PQ_ENTRY_SIZE) 150 151 #define HBM_DMA_QMAN_LENGTH 4096 152 #define HBM_DMA_QMAN_SIZE_IN_BYTES \ 153 (HBM_DMA_QMAN_LENGTH * QMAN_PQ_ENTRY_SIZE) 154 155 #define TPC_QMAN_LENGTH 1024 156 #define TPC_QMAN_SIZE_IN_BYTES (TPC_QMAN_LENGTH * QMAN_PQ_ENTRY_SIZE) 157 158 #define NIC_QMAN_LENGTH 4096 159 #define NIC_QMAN_SIZE_IN_BYTES (NIC_QMAN_LENGTH * QMAN_PQ_ENTRY_SIZE) 160 161 162 #define SRAM_USER_BASE_OFFSET GAUDI_DRIVER_SRAM_RESERVED_SIZE_FROM_START 163 164 /* Virtual address space */ 165 #define VA_HOST_SPACE_START 0x1000000000000ull /* 256TB */ 166 #define VA_HOST_SPACE_END 0x3FF8000000000ull /* 1PB - 512GB */ 167 #define VA_HOST_SPACE_SIZE (VA_HOST_SPACE_END - \ 168 VA_HOST_SPACE_START) /* 767TB */ 169 #define HOST_SPACE_INTERNAL_CB_SZ SZ_2M 170 171 #define HW_CAP_PLL BIT(0) 172 #define HW_CAP_HBM BIT(1) 173 #define HW_CAP_MMU BIT(2) 174 #define HW_CAP_MME BIT(3) 175 #define HW_CAP_CPU BIT(4) 176 #define HW_CAP_PCI_DMA BIT(5) 177 #define HW_CAP_MSI BIT(6) 178 #define HW_CAP_CPU_Q BIT(7) 179 #define HW_CAP_HBM_DMA BIT(8) 180 #define HW_CAP_SRAM_SCRAMBLER BIT(10) 181 #define HW_CAP_HBM_SCRAMBLER BIT(11) 182 183 #define HW_CAP_NIC0 BIT(14) 184 #define HW_CAP_NIC1 BIT(15) 185 #define HW_CAP_NIC2 BIT(16) 186 #define HW_CAP_NIC3 BIT(17) 187 #define HW_CAP_NIC4 BIT(18) 188 #define HW_CAP_NIC5 BIT(19) 189 #define HW_CAP_NIC6 BIT(20) 190 #define HW_CAP_NIC7 BIT(21) 191 #define HW_CAP_NIC8 BIT(22) 192 #define HW_CAP_NIC9 BIT(23) 193 #define HW_CAP_NIC_MASK GENMASK(23, 14) 194 #define HW_CAP_NIC_SHIFT 14 195 196 #define HW_CAP_TPC0 BIT(24) 197 #define HW_CAP_TPC1 BIT(25) 198 #define HW_CAP_TPC2 BIT(26) 199 #define HW_CAP_TPC3 BIT(27) 200 #define HW_CAP_TPC4 BIT(28) 201 #define HW_CAP_TPC5 BIT(29) 202 #define HW_CAP_TPC6 BIT(30) 203 #define HW_CAP_TPC7 BIT(31) 204 #define HW_CAP_TPC_MASK GENMASK(31, 24) 205 #define HW_CAP_TPC_SHIFT 24 206 207 #define NEXT_SYNC_OBJ_ADDR_INTERVAL \ 208 (mmSYNC_MNGR_W_N_SYNC_MNGR_OBJS_SOB_OBJ_0 - \ 209 mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0) 210 #define NUM_OF_MME_ENGINES 2 211 #define NUM_OF_MME_SUB_ENGINES 2 212 #define NUM_OF_TPC_ENGINES 8 213 #define NUM_OF_DMA_ENGINES 8 214 #define NUM_OF_QUEUES 5 215 #define NUM_OF_STREAMS 4 216 #define NUM_OF_FENCES 4 217 218 219 #define GAUDI_CPU_PCI_MSB_ADDR(addr) (((addr) & GENMASK_ULL(49, 39)) >> 39) 220 #define GAUDI_PCI_TO_CPU_ADDR(addr) \ 221 do { \ 222 (addr) &= ~GENMASK_ULL(49, 39); \ 223 (addr) |= BIT_ULL(39); \ 224 } while (0) 225 #define GAUDI_CPU_TO_PCI_ADDR(addr, extension) \ 226 do { \ 227 (addr) &= ~GENMASK_ULL(49, 39); \ 228 (addr) |= (u64) (extension) << 39; \ 229 } while (0) 230 231 enum gaudi_dma_channels { 232 GAUDI_PCI_DMA_1, 233 GAUDI_PCI_DMA_2, 234 GAUDI_HBM_DMA_1, 235 GAUDI_HBM_DMA_2, 236 GAUDI_HBM_DMA_3, 237 GAUDI_HBM_DMA_4, 238 GAUDI_HBM_DMA_5, 239 GAUDI_HBM_DMA_6, 240 GAUDI_DMA_MAX 241 }; 242 243 enum gaudi_tpc_mask { 244 GAUDI_TPC_MASK_TPC0 = 0x01, 245 GAUDI_TPC_MASK_TPC1 = 0x02, 246 GAUDI_TPC_MASK_TPC2 = 0x04, 247 GAUDI_TPC_MASK_TPC3 = 0x08, 248 GAUDI_TPC_MASK_TPC4 = 0x10, 249 GAUDI_TPC_MASK_TPC5 = 0x20, 250 GAUDI_TPC_MASK_TPC6 = 0x40, 251 GAUDI_TPC_MASK_TPC7 = 0x80, 252 GAUDI_TPC_MASK_ALL = 0xFF 253 }; 254 255 enum gaudi_nic_mask { 256 GAUDI_NIC_MASK_NIC0 = 0x01, 257 GAUDI_NIC_MASK_NIC1 = 0x02, 258 GAUDI_NIC_MASK_NIC2 = 0x04, 259 GAUDI_NIC_MASK_NIC3 = 0x08, 260 GAUDI_NIC_MASK_NIC4 = 0x10, 261 GAUDI_NIC_MASK_NIC5 = 0x20, 262 GAUDI_NIC_MASK_NIC6 = 0x40, 263 GAUDI_NIC_MASK_NIC7 = 0x80, 264 GAUDI_NIC_MASK_NIC8 = 0x100, 265 GAUDI_NIC_MASK_NIC9 = 0x200, 266 GAUDI_NIC_MASK_ALL = 0x3FF 267 }; 268 269 /* 270 * struct gaudi_hw_sob_group - H/W SOB group info. 271 * @hdev: habanalabs device structure. 272 * @kref: refcount of this SOB group. group will reset once refcount is zero. 273 * @base_sob_id: base sob id of this SOB group. 274 * @queue_id: id of the queue that waits on this sob group 275 */ 276 struct gaudi_hw_sob_group { 277 struct hl_device *hdev; 278 struct kref kref; 279 u32 base_sob_id; 280 u32 queue_id; 281 }; 282 283 #define NUM_SOB_GROUPS (HL_RSVD_SOBS * QMAN_STREAMS) 284 /** 285 * struct gaudi_collective_properties - 286 * holds all SOB groups and queues info reserved for the collective 287 * @hw_sob_group: H/W SOB groups. 288 * @next_sob_group_val: the next value to use for the currently used SOB group. 289 * @curr_sob_group_idx: the index of the currently used SOB group. 290 * @mstr_sob_mask: pre-defined masks for collective master monitors 291 */ 292 struct gaudi_collective_properties { 293 struct gaudi_hw_sob_group hw_sob_group[NUM_SOB_GROUPS]; 294 u16 next_sob_group_val[QMAN_STREAMS]; 295 u8 curr_sob_group_idx[QMAN_STREAMS]; 296 u8 mstr_sob_mask[HL_COLLECTIVE_RSVD_MSTR_MONS]; 297 }; 298 299 /** 300 * struct gaudi_internal_qman_info - Internal QMAN information. 301 * @pq_kernel_addr: Kernel address of the PQ memory area in the host. 302 * @pq_dma_addr: DMA address of the PQ memory area in the host. 303 * @pq_size: Size of allocated host memory for PQ. 304 */ 305 struct gaudi_internal_qman_info { 306 void *pq_kernel_addr; 307 dma_addr_t pq_dma_addr; 308 size_t pq_size; 309 }; 310 311 /** 312 * struct gaudi_device - ASIC specific manage structure. 313 * @cpucp_info_get: get information on device from CPU-CP 314 * @hw_queues_lock: protects the H/W queues from concurrent access. 315 * @internal_qmans: Internal QMANs information. The array size is larger than 316 * the actual number of internal queues because they are not in 317 * consecutive order. 318 * @hbm_bar_cur_addr: current address of HBM PCI bar. 319 * @events: array that holds all event id's 320 * @events_stat: array that holds histogram of all received events. 321 * @events_stat_aggregate: same as events_stat but doesn't get cleared on reset 322 * @hw_cap_initialized: This field contains a bit per H/W engine. When that 323 * engine is initialized, that bit is set by the driver to 324 * signal we can use this engine in later code paths. 325 * Each bit is cleared upon reset of its corresponding H/W 326 * engine. 327 * @multi_msi_mode: whether we are working in multi MSI single MSI mode. 328 * Multi MSI is possible only with IOMMU enabled. 329 * @mmu_cache_inv_pi: PI for MMU cache invalidation flow. The H/W expects an 330 * 8-bit value so use u8. 331 */ 332 struct gaudi_device { 333 int (*cpucp_info_get)(struct hl_device *hdev); 334 335 /* TODO: remove hw_queues_lock after moving to scheduler code */ 336 spinlock_t hw_queues_lock; 337 338 struct gaudi_internal_qman_info internal_qmans[GAUDI_QUEUE_ID_SIZE]; 339 340 struct gaudi_collective_properties collective_props; 341 342 u64 hbm_bar_cur_addr; 343 344 u32 events[GAUDI_EVENT_SIZE]; 345 u32 events_stat[GAUDI_EVENT_SIZE]; 346 u32 events_stat_aggregate[GAUDI_EVENT_SIZE]; 347 u32 hw_cap_initialized; 348 u8 multi_msi_mode; 349 u8 mmu_cache_inv_pi; 350 }; 351 352 void gaudi_init_security(struct hl_device *hdev); 353 void gaudi_ack_protection_bits_errors(struct hl_device *hdev); 354 int gaudi_debug_coresight(struct hl_device *hdev, struct hl_ctx *ctx, void *data); 355 void gaudi_halt_coresight(struct hl_device *hdev, struct hl_ctx *ctx); 356 void gaudi_mmu_prepare_reg(struct hl_device *hdev, u64 reg, u32 asid); 357 358 #endif /* GAUDIP_H_ */ 359