1 // SPDX-License-Identifier: GPL-2.0
2
3 /*
4 * Copyright 2016-2021 HabanaLabs, Ltd.
5 * All Rights Reserved.
6 */
7
8 #include "habanalabs.h"
9 #include "../include/common/hl_boot_if.h"
10
11 #include <linux/firmware.h>
12 #include <linux/crc32.h>
13 #include <linux/slab.h>
14 #include <linux/ctype.h>
15
16 #define FW_FILE_MAX_SIZE 0x1400000 /* maximum size of 20MB */
17
18 #define FW_CPU_STATUS_POLL_INTERVAL_USEC 10000
19
extract_fw_ver_from_str(const char * fw_str)20 static char *extract_fw_ver_from_str(const char *fw_str)
21 {
22 char *str, *fw_ver, *whitespace;
23
24 fw_ver = kmalloc(16, GFP_KERNEL);
25 if (!fw_ver)
26 return NULL;
27
28 str = strnstr(fw_str, "fw-", VERSION_MAX_LEN);
29 if (!str)
30 goto free_fw_ver;
31
32 /* Skip the fw- part */
33 str += 3;
34
35 /* Copy until the next whitespace */
36 whitespace = strnstr(str, " ", 15);
37 if (!whitespace)
38 goto free_fw_ver;
39
40 strscpy(fw_ver, str, whitespace - str + 1);
41
42 return fw_ver;
43
44 free_fw_ver:
45 kfree(fw_ver);
46 return NULL;
47 }
48
hl_request_fw(struct hl_device * hdev,const struct firmware ** firmware_p,const char * fw_name)49 static int hl_request_fw(struct hl_device *hdev,
50 const struct firmware **firmware_p,
51 const char *fw_name)
52 {
53 size_t fw_size;
54 int rc;
55
56 rc = request_firmware(firmware_p, fw_name, hdev->dev);
57 if (rc) {
58 dev_err(hdev->dev, "Firmware file %s is not found! (error %d)\n",
59 fw_name, rc);
60 goto out;
61 }
62
63 fw_size = (*firmware_p)->size;
64 if ((fw_size % 4) != 0) {
65 dev_err(hdev->dev, "Illegal %s firmware size %zu\n",
66 fw_name, fw_size);
67 rc = -EINVAL;
68 goto release_fw;
69 }
70
71 dev_dbg(hdev->dev, "%s firmware size == %zu\n", fw_name, fw_size);
72
73 if (fw_size > FW_FILE_MAX_SIZE) {
74 dev_err(hdev->dev,
75 "FW file size %zu exceeds maximum of %u bytes\n",
76 fw_size, FW_FILE_MAX_SIZE);
77 rc = -EINVAL;
78 goto release_fw;
79 }
80
81 return 0;
82
83 release_fw:
84 release_firmware(*firmware_p);
85 out:
86 return rc;
87 }
88
89 /**
90 * hl_release_firmware() - release FW
91 *
92 * @fw: fw descriptor
93 *
94 * note: this inline function added to serve as a comprehensive mirror for the
95 * hl_request_fw function.
96 */
hl_release_firmware(const struct firmware * fw)97 static inline void hl_release_firmware(const struct firmware *fw)
98 {
99 release_firmware(fw);
100 }
101
102 /**
103 * hl_fw_copy_fw_to_device() - copy FW to device
104 *
105 * @hdev: pointer to hl_device structure.
106 * @fw: fw descriptor
107 * @dst: IO memory mapped address space to copy firmware to
108 * @src_offset: offset in src FW to copy from
109 * @size: amount of bytes to copy (0 to copy the whole binary)
110 *
111 * actual copy of FW binary data to device, shared by static and dynamic loaders
112 */
hl_fw_copy_fw_to_device(struct hl_device * hdev,const struct firmware * fw,void __iomem * dst,u32 src_offset,u32 size)113 static int hl_fw_copy_fw_to_device(struct hl_device *hdev,
114 const struct firmware *fw, void __iomem *dst,
115 u32 src_offset, u32 size)
116 {
117 const void *fw_data;
118
119 /* size 0 indicates to copy the whole file */
120 if (!size)
121 size = fw->size;
122
123 if (src_offset + size > fw->size) {
124 dev_err(hdev->dev,
125 "size to copy(%u) and offset(%u) are invalid\n",
126 size, src_offset);
127 return -EINVAL;
128 }
129
130 fw_data = (const void *) fw->data;
131
132 memcpy_toio(dst, fw_data + src_offset, size);
133 return 0;
134 }
135
136 /**
137 * hl_fw_copy_msg_to_device() - copy message to device
138 *
139 * @hdev: pointer to hl_device structure.
140 * @msg: message
141 * @dst: IO memory mapped address space to copy firmware to
142 * @src_offset: offset in src message to copy from
143 * @size: amount of bytes to copy (0 to copy the whole binary)
144 *
145 * actual copy of message data to device.
146 */
hl_fw_copy_msg_to_device(struct hl_device * hdev,struct lkd_msg_comms * msg,void __iomem * dst,u32 src_offset,u32 size)147 static int hl_fw_copy_msg_to_device(struct hl_device *hdev,
148 struct lkd_msg_comms *msg, void __iomem *dst,
149 u32 src_offset, u32 size)
150 {
151 void *msg_data;
152
153 /* size 0 indicates to copy the whole file */
154 if (!size)
155 size = sizeof(struct lkd_msg_comms);
156
157 if (src_offset + size > sizeof(struct lkd_msg_comms)) {
158 dev_err(hdev->dev,
159 "size to copy(%u) and offset(%u) are invalid\n",
160 size, src_offset);
161 return -EINVAL;
162 }
163
164 msg_data = (void *) msg;
165
166 memcpy_toio(dst, msg_data + src_offset, size);
167
168 return 0;
169 }
170
171 /**
172 * hl_fw_load_fw_to_device() - Load F/W code to device's memory.
173 *
174 * @hdev: pointer to hl_device structure.
175 * @fw_name: the firmware image name
176 * @dst: IO memory mapped address space to copy firmware to
177 * @src_offset: offset in src FW to copy from
178 * @size: amount of bytes to copy (0 to copy the whole binary)
179 *
180 * Copy fw code from firmware file to device memory.
181 *
182 * Return: 0 on success, non-zero for failure.
183 */
hl_fw_load_fw_to_device(struct hl_device * hdev,const char * fw_name,void __iomem * dst,u32 src_offset,u32 size)184 int hl_fw_load_fw_to_device(struct hl_device *hdev, const char *fw_name,
185 void __iomem *dst, u32 src_offset, u32 size)
186 {
187 const struct firmware *fw;
188 int rc;
189
190 rc = hl_request_fw(hdev, &fw, fw_name);
191 if (rc)
192 return rc;
193
194 rc = hl_fw_copy_fw_to_device(hdev, fw, dst, src_offset, size);
195
196 hl_release_firmware(fw);
197 return rc;
198 }
199
hl_fw_send_pci_access_msg(struct hl_device * hdev,u32 opcode)200 int hl_fw_send_pci_access_msg(struct hl_device *hdev, u32 opcode)
201 {
202 struct cpucp_packet pkt = {};
203
204 pkt.ctl = cpu_to_le32(opcode << CPUCP_PKT_CTL_OPCODE_SHIFT);
205
206 return hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt,
207 sizeof(pkt), 0, NULL);
208 }
209
hl_fw_send_cpu_message(struct hl_device * hdev,u32 hw_queue_id,u32 * msg,u16 len,u32 timeout,u64 * result)210 int hl_fw_send_cpu_message(struct hl_device *hdev, u32 hw_queue_id, u32 *msg,
211 u16 len, u32 timeout, u64 *result)
212 {
213 struct hl_hw_queue *queue = &hdev->kernel_queues[hw_queue_id];
214 struct asic_fixed_properties *prop = &hdev->asic_prop;
215 struct cpucp_packet *pkt;
216 dma_addr_t pkt_dma_addr;
217 u32 tmp, expected_ack_val;
218 int rc = 0;
219
220 pkt = hdev->asic_funcs->cpu_accessible_dma_pool_alloc(hdev, len,
221 &pkt_dma_addr);
222 if (!pkt) {
223 dev_err(hdev->dev,
224 "Failed to allocate DMA memory for packet to CPU\n");
225 return -ENOMEM;
226 }
227
228 memcpy(pkt, msg, len);
229
230 mutex_lock(&hdev->send_cpu_message_lock);
231
232 if (hdev->disabled)
233 goto out;
234
235 if (hdev->device_cpu_disabled) {
236 rc = -EIO;
237 goto out;
238 }
239
240 /* set fence to a non valid value */
241 pkt->fence = cpu_to_le32(UINT_MAX);
242
243 /*
244 * The CPU queue is a synchronous queue with an effective depth of
245 * a single entry (although it is allocated with room for multiple
246 * entries). We lock on it using 'send_cpu_message_lock' which
247 * serializes accesses to the CPU queue.
248 * Which means that we don't need to lock the access to the entire H/W
249 * queues module when submitting a JOB to the CPU queue.
250 */
251 hl_hw_queue_submit_bd(hdev, queue, 0, len, pkt_dma_addr);
252
253 if (prop->fw_app_cpu_boot_dev_sts0 & CPU_BOOT_DEV_STS0_PKT_PI_ACK_EN)
254 expected_ack_val = queue->pi;
255 else
256 expected_ack_val = CPUCP_PACKET_FENCE_VAL;
257
258 rc = hl_poll_timeout_memory(hdev, &pkt->fence, tmp,
259 (tmp == expected_ack_val), 1000,
260 timeout, true);
261
262 hl_hw_queue_inc_ci_kernel(hdev, hw_queue_id);
263
264 if (rc == -ETIMEDOUT) {
265 dev_err(hdev->dev, "Device CPU packet timeout (0x%x)\n", tmp);
266 hdev->device_cpu_disabled = true;
267 goto out;
268 }
269
270 tmp = le32_to_cpu(pkt->ctl);
271
272 rc = (tmp & CPUCP_PKT_CTL_RC_MASK) >> CPUCP_PKT_CTL_RC_SHIFT;
273 if (rc) {
274 dev_err(hdev->dev, "F/W ERROR %d for CPU packet %d\n",
275 rc,
276 (tmp & CPUCP_PKT_CTL_OPCODE_MASK)
277 >> CPUCP_PKT_CTL_OPCODE_SHIFT);
278 rc = -EIO;
279 } else if (result) {
280 *result = le64_to_cpu(pkt->result);
281 }
282
283 out:
284 mutex_unlock(&hdev->send_cpu_message_lock);
285
286 hdev->asic_funcs->cpu_accessible_dma_pool_free(hdev, len, pkt);
287
288 return rc;
289 }
290
hl_fw_unmask_irq(struct hl_device * hdev,u16 event_type)291 int hl_fw_unmask_irq(struct hl_device *hdev, u16 event_type)
292 {
293 struct cpucp_packet pkt;
294 u64 result;
295 int rc;
296
297 memset(&pkt, 0, sizeof(pkt));
298
299 pkt.ctl = cpu_to_le32(CPUCP_PACKET_UNMASK_RAZWI_IRQ <<
300 CPUCP_PKT_CTL_OPCODE_SHIFT);
301 pkt.value = cpu_to_le64(event_type);
302
303 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
304 0, &result);
305
306 if (rc)
307 dev_err(hdev->dev, "failed to unmask RAZWI IRQ %d", event_type);
308
309 return rc;
310 }
311
hl_fw_unmask_irq_arr(struct hl_device * hdev,const u32 * irq_arr,size_t irq_arr_size)312 int hl_fw_unmask_irq_arr(struct hl_device *hdev, const u32 *irq_arr,
313 size_t irq_arr_size)
314 {
315 struct cpucp_unmask_irq_arr_packet *pkt;
316 size_t total_pkt_size;
317 u64 result;
318 int rc;
319
320 total_pkt_size = sizeof(struct cpucp_unmask_irq_arr_packet) +
321 irq_arr_size;
322
323 /* data should be aligned to 8 bytes in order to CPU-CP to copy it */
324 total_pkt_size = (total_pkt_size + 0x7) & ~0x7;
325
326 /* total_pkt_size is casted to u16 later on */
327 if (total_pkt_size > USHRT_MAX) {
328 dev_err(hdev->dev, "too many elements in IRQ array\n");
329 return -EINVAL;
330 }
331
332 pkt = kzalloc(total_pkt_size, GFP_KERNEL);
333 if (!pkt)
334 return -ENOMEM;
335
336 pkt->length = cpu_to_le32(irq_arr_size / sizeof(irq_arr[0]));
337 memcpy(&pkt->irqs, irq_arr, irq_arr_size);
338
339 pkt->cpucp_pkt.ctl = cpu_to_le32(CPUCP_PACKET_UNMASK_RAZWI_IRQ_ARRAY <<
340 CPUCP_PKT_CTL_OPCODE_SHIFT);
341
342 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) pkt,
343 total_pkt_size, 0, &result);
344
345 if (rc)
346 dev_err(hdev->dev, "failed to unmask IRQ array\n");
347
348 kfree(pkt);
349
350 return rc;
351 }
352
hl_fw_test_cpu_queue(struct hl_device * hdev)353 int hl_fw_test_cpu_queue(struct hl_device *hdev)
354 {
355 struct cpucp_packet test_pkt = {};
356 u64 result;
357 int rc;
358
359 test_pkt.ctl = cpu_to_le32(CPUCP_PACKET_TEST <<
360 CPUCP_PKT_CTL_OPCODE_SHIFT);
361 test_pkt.value = cpu_to_le64(CPUCP_PACKET_FENCE_VAL);
362
363 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &test_pkt,
364 sizeof(test_pkt), 0, &result);
365
366 if (!rc) {
367 if (result != CPUCP_PACKET_FENCE_VAL)
368 dev_err(hdev->dev,
369 "CPU queue test failed (%#08llx)\n", result);
370 } else {
371 dev_err(hdev->dev, "CPU queue test failed, error %d\n", rc);
372 }
373
374 return rc;
375 }
376
hl_fw_cpu_accessible_dma_pool_alloc(struct hl_device * hdev,size_t size,dma_addr_t * dma_handle)377 void *hl_fw_cpu_accessible_dma_pool_alloc(struct hl_device *hdev, size_t size,
378 dma_addr_t *dma_handle)
379 {
380 u64 kernel_addr;
381
382 kernel_addr = gen_pool_alloc(hdev->cpu_accessible_dma_pool, size);
383
384 *dma_handle = hdev->cpu_accessible_dma_address +
385 (kernel_addr - (u64) (uintptr_t) hdev->cpu_accessible_dma_mem);
386
387 return (void *) (uintptr_t) kernel_addr;
388 }
389
hl_fw_cpu_accessible_dma_pool_free(struct hl_device * hdev,size_t size,void * vaddr)390 void hl_fw_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size,
391 void *vaddr)
392 {
393 gen_pool_free(hdev->cpu_accessible_dma_pool, (u64) (uintptr_t) vaddr,
394 size);
395 }
396
hl_fw_send_heartbeat(struct hl_device * hdev)397 int hl_fw_send_heartbeat(struct hl_device *hdev)
398 {
399 struct cpucp_packet hb_pkt;
400 u64 result;
401 int rc;
402
403 memset(&hb_pkt, 0, sizeof(hb_pkt));
404 hb_pkt.ctl = cpu_to_le32(CPUCP_PACKET_TEST <<
405 CPUCP_PKT_CTL_OPCODE_SHIFT);
406 hb_pkt.value = cpu_to_le64(CPUCP_PACKET_FENCE_VAL);
407
408 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &hb_pkt,
409 sizeof(hb_pkt), 0, &result);
410
411 if ((rc) || (result != CPUCP_PACKET_FENCE_VAL))
412 return -EIO;
413
414 if (le32_to_cpu(hb_pkt.status_mask) &
415 CPUCP_PKT_HB_STATUS_EQ_FAULT_MASK) {
416 dev_warn(hdev->dev, "FW reported EQ fault during heartbeat\n");
417 rc = -EIO;
418 }
419
420 return rc;
421 }
422
fw_report_boot_dev0(struct hl_device * hdev,u32 err_val,u32 sts_val)423 static bool fw_report_boot_dev0(struct hl_device *hdev, u32 err_val,
424 u32 sts_val)
425 {
426 bool err_exists = false;
427
428 if (!(err_val & CPU_BOOT_ERR0_ENABLED))
429 return false;
430
431 if (err_val & CPU_BOOT_ERR0_DRAM_INIT_FAIL) {
432 dev_err(hdev->dev,
433 "Device boot error - DRAM initialization failed\n");
434 err_exists = true;
435 }
436
437 if (err_val & CPU_BOOT_ERR0_FIT_CORRUPTED) {
438 dev_err(hdev->dev, "Device boot error - FIT image corrupted\n");
439 err_exists = true;
440 }
441
442 if (err_val & CPU_BOOT_ERR0_TS_INIT_FAIL) {
443 dev_err(hdev->dev,
444 "Device boot error - Thermal Sensor initialization failed\n");
445 err_exists = true;
446 }
447
448 if (err_val & CPU_BOOT_ERR0_DRAM_SKIPPED) {
449 dev_warn(hdev->dev,
450 "Device boot warning - Skipped DRAM initialization\n");
451 /* This is a warning so we don't want it to disable the
452 * device
453 */
454 err_val &= ~CPU_BOOT_ERR0_DRAM_SKIPPED;
455 }
456
457 if (err_val & CPU_BOOT_ERR0_BMC_WAIT_SKIPPED) {
458 if (hdev->bmc_enable) {
459 dev_err(hdev->dev,
460 "Device boot error - Skipped waiting for BMC\n");
461 err_exists = true;
462 } else {
463 dev_info(hdev->dev,
464 "Device boot message - Skipped waiting for BMC\n");
465 /* This is an info so we don't want it to disable the
466 * device
467 */
468 err_val &= ~CPU_BOOT_ERR0_BMC_WAIT_SKIPPED;
469 }
470 }
471
472 if (err_val & CPU_BOOT_ERR0_NIC_DATA_NOT_RDY) {
473 dev_err(hdev->dev,
474 "Device boot error - Serdes data from BMC not available\n");
475 err_exists = true;
476 }
477
478 if (err_val & CPU_BOOT_ERR0_NIC_FW_FAIL) {
479 dev_err(hdev->dev,
480 "Device boot error - NIC F/W initialization failed\n");
481 err_exists = true;
482 }
483
484 if (err_val & CPU_BOOT_ERR0_SECURITY_NOT_RDY) {
485 dev_err(hdev->dev,
486 "Device boot warning - security not ready\n");
487 err_exists = true;
488 }
489
490 if (err_val & CPU_BOOT_ERR0_SECURITY_FAIL) {
491 dev_err(hdev->dev, "Device boot error - security failure\n");
492 err_exists = true;
493 }
494
495 if (err_val & CPU_BOOT_ERR0_EFUSE_FAIL) {
496 dev_err(hdev->dev, "Device boot error - eFuse failure\n");
497 err_exists = true;
498 }
499
500 if (err_val & CPU_BOOT_ERR0_PRI_IMG_VER_FAIL) {
501 dev_warn(hdev->dev,
502 "Device boot warning - Failed to load preboot primary image\n");
503 /* This is a warning so we don't want it to disable the
504 * device as we have a secondary preboot image
505 */
506 err_val &= ~CPU_BOOT_ERR0_PRI_IMG_VER_FAIL;
507 }
508
509 if (err_val & CPU_BOOT_ERR0_SEC_IMG_VER_FAIL) {
510 dev_err(hdev->dev, "Device boot error - Failed to load preboot secondary image\n");
511 err_exists = true;
512 }
513
514 if (err_val & CPU_BOOT_ERR0_PLL_FAIL) {
515 dev_err(hdev->dev, "Device boot error - PLL failure\n");
516 err_exists = true;
517 }
518
519 if (err_val & CPU_BOOT_ERR0_DEVICE_UNUSABLE_FAIL) {
520 /* Ignore this bit, don't prevent driver loading */
521 dev_dbg(hdev->dev, "device unusable status is set\n");
522 err_val &= ~CPU_BOOT_ERR0_DEVICE_UNUSABLE_FAIL;
523 }
524
525 if (sts_val & CPU_BOOT_DEV_STS0_ENABLED)
526 dev_dbg(hdev->dev, "Device status0 %#x\n", sts_val);
527
528 if (!err_exists && (err_val & ~CPU_BOOT_ERR0_ENABLED)) {
529 dev_err(hdev->dev,
530 "Device boot error - unknown ERR0 error 0x%08x\n", err_val);
531 err_exists = true;
532 }
533
534 /* return error only if it's in the predefined mask */
535 if (err_exists && ((err_val & ~CPU_BOOT_ERR0_ENABLED) &
536 lower_32_bits(hdev->boot_error_status_mask)))
537 return true;
538
539 return false;
540 }
541
542 /* placeholder for ERR1 as no errors defined there yet */
fw_report_boot_dev1(struct hl_device * hdev,u32 err_val,u32 sts_val)543 static bool fw_report_boot_dev1(struct hl_device *hdev, u32 err_val,
544 u32 sts_val)
545 {
546 /*
547 * keep this variable to preserve the logic of the function.
548 * this way it would require less modifications when error will be
549 * added to DEV_ERR1
550 */
551 bool err_exists = false;
552
553 if (!(err_val & CPU_BOOT_ERR1_ENABLED))
554 return false;
555
556 if (sts_val & CPU_BOOT_DEV_STS1_ENABLED)
557 dev_dbg(hdev->dev, "Device status1 %#x\n", sts_val);
558
559 if (!err_exists && (err_val & ~CPU_BOOT_ERR1_ENABLED)) {
560 dev_err(hdev->dev,
561 "Device boot error - unknown ERR1 error 0x%08x\n",
562 err_val);
563 err_exists = true;
564 }
565
566 /* return error only if it's in the predefined mask */
567 if (err_exists && ((err_val & ~CPU_BOOT_ERR1_ENABLED) &
568 upper_32_bits(hdev->boot_error_status_mask)))
569 return true;
570
571 return false;
572 }
573
fw_read_errors(struct hl_device * hdev,u32 boot_err0_reg,u32 boot_err1_reg,u32 cpu_boot_dev_status0_reg,u32 cpu_boot_dev_status1_reg)574 static int fw_read_errors(struct hl_device *hdev, u32 boot_err0_reg,
575 u32 boot_err1_reg, u32 cpu_boot_dev_status0_reg,
576 u32 cpu_boot_dev_status1_reg)
577 {
578 u32 err_val, status_val;
579 bool err_exists = false;
580
581 /* Some of the firmware status codes are deprecated in newer f/w
582 * versions. In those versions, the errors are reported
583 * in different registers. Therefore, we need to check those
584 * registers and print the exact errors. Moreover, there
585 * may be multiple errors, so we need to report on each error
586 * separately. Some of the error codes might indicate a state
587 * that is not an error per-se, but it is an error in production
588 * environment
589 */
590 err_val = RREG32(boot_err0_reg);
591 status_val = RREG32(cpu_boot_dev_status0_reg);
592 err_exists = fw_report_boot_dev0(hdev, err_val, status_val);
593
594 err_val = RREG32(boot_err1_reg);
595 status_val = RREG32(cpu_boot_dev_status1_reg);
596 err_exists |= fw_report_boot_dev1(hdev, err_val, status_val);
597
598 if (err_exists)
599 return -EIO;
600
601 return 0;
602 }
603
hl_fw_cpucp_info_get(struct hl_device * hdev,u32 sts_boot_dev_sts0_reg,u32 sts_boot_dev_sts1_reg,u32 boot_err0_reg,u32 boot_err1_reg)604 int hl_fw_cpucp_info_get(struct hl_device *hdev,
605 u32 sts_boot_dev_sts0_reg,
606 u32 sts_boot_dev_sts1_reg, u32 boot_err0_reg,
607 u32 boot_err1_reg)
608 {
609 struct asic_fixed_properties *prop = &hdev->asic_prop;
610 struct cpucp_packet pkt = {};
611 dma_addr_t cpucp_info_dma_addr;
612 void *cpucp_info_cpu_addr;
613 char *kernel_ver;
614 u64 result;
615 int rc;
616
617 cpucp_info_cpu_addr =
618 hdev->asic_funcs->cpu_accessible_dma_pool_alloc(hdev,
619 sizeof(struct cpucp_info),
620 &cpucp_info_dma_addr);
621 if (!cpucp_info_cpu_addr) {
622 dev_err(hdev->dev,
623 "Failed to allocate DMA memory for CPU-CP info packet\n");
624 return -ENOMEM;
625 }
626
627 memset(cpucp_info_cpu_addr, 0, sizeof(struct cpucp_info));
628
629 pkt.ctl = cpu_to_le32(CPUCP_PACKET_INFO_GET <<
630 CPUCP_PKT_CTL_OPCODE_SHIFT);
631 pkt.addr = cpu_to_le64(cpucp_info_dma_addr);
632 pkt.data_max_size = cpu_to_le32(sizeof(struct cpucp_info));
633
634 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
635 HL_CPUCP_INFO_TIMEOUT_USEC, &result);
636 if (rc) {
637 dev_err(hdev->dev,
638 "Failed to handle CPU-CP info pkt, error %d\n", rc);
639 goto out;
640 }
641
642 rc = fw_read_errors(hdev, boot_err0_reg, boot_err1_reg,
643 sts_boot_dev_sts0_reg, sts_boot_dev_sts1_reg);
644 if (rc) {
645 dev_err(hdev->dev, "Errors in device boot\n");
646 goto out;
647 }
648
649 memcpy(&prop->cpucp_info, cpucp_info_cpu_addr,
650 sizeof(prop->cpucp_info));
651
652 rc = hl_build_hwmon_channel_info(hdev, prop->cpucp_info.sensors);
653 if (rc) {
654 dev_err(hdev->dev,
655 "Failed to build hwmon channel info, error %d\n", rc);
656 rc = -EFAULT;
657 goto out;
658 }
659
660 kernel_ver = extract_fw_ver_from_str(prop->cpucp_info.kernel_version);
661 if (kernel_ver) {
662 dev_info(hdev->dev, "Linux version %s", kernel_ver);
663 kfree(kernel_ver);
664 }
665
666 /* assume EQ code doesn't need to check eqe index */
667 hdev->event_queue.check_eqe_index = false;
668
669 /* Read FW application security bits again */
670 if (prop->fw_cpu_boot_dev_sts0_valid) {
671 prop->fw_app_cpu_boot_dev_sts0 = RREG32(sts_boot_dev_sts0_reg);
672 if (prop->fw_app_cpu_boot_dev_sts0 &
673 CPU_BOOT_DEV_STS0_EQ_INDEX_EN)
674 hdev->event_queue.check_eqe_index = true;
675 }
676
677 if (prop->fw_cpu_boot_dev_sts1_valid)
678 prop->fw_app_cpu_boot_dev_sts1 = RREG32(sts_boot_dev_sts1_reg);
679
680 out:
681 hdev->asic_funcs->cpu_accessible_dma_pool_free(hdev,
682 sizeof(struct cpucp_info), cpucp_info_cpu_addr);
683
684 return rc;
685 }
686
hl_fw_send_msi_info_msg(struct hl_device * hdev)687 static int hl_fw_send_msi_info_msg(struct hl_device *hdev)
688 {
689 struct cpucp_array_data_packet *pkt;
690 size_t total_pkt_size, data_size;
691 u64 result;
692 int rc;
693
694 /* skip sending this info for unsupported ASICs */
695 if (!hdev->asic_funcs->get_msi_info)
696 return 0;
697
698 data_size = CPUCP_NUM_OF_MSI_TYPES * sizeof(u32);
699 total_pkt_size = sizeof(struct cpucp_array_data_packet) + data_size;
700
701 /* data should be aligned to 8 bytes in order to CPU-CP to copy it */
702 total_pkt_size = (total_pkt_size + 0x7) & ~0x7;
703
704 /* total_pkt_size is casted to u16 later on */
705 if (total_pkt_size > USHRT_MAX) {
706 dev_err(hdev->dev, "CPUCP array data is too big\n");
707 return -EINVAL;
708 }
709
710 pkt = kzalloc(total_pkt_size, GFP_KERNEL);
711 if (!pkt)
712 return -ENOMEM;
713
714 pkt->length = cpu_to_le32(CPUCP_NUM_OF_MSI_TYPES);
715
716 memset((void *) &pkt->data, 0xFF, data_size);
717 hdev->asic_funcs->get_msi_info(pkt->data);
718
719 pkt->cpucp_pkt.ctl = cpu_to_le32(CPUCP_PACKET_MSI_INFO_SET <<
720 CPUCP_PKT_CTL_OPCODE_SHIFT);
721
722 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *)pkt,
723 total_pkt_size, 0, &result);
724
725 /*
726 * in case packet result is invalid it means that FW does not support
727 * this feature and will use default/hard coded MSI values. no reason
728 * to stop the boot
729 */
730 if (rc && result == cpucp_packet_invalid)
731 rc = 0;
732
733 if (rc)
734 dev_err(hdev->dev, "failed to send CPUCP array data\n");
735
736 kfree(pkt);
737
738 return rc;
739 }
740
hl_fw_cpucp_handshake(struct hl_device * hdev,u32 sts_boot_dev_sts0_reg,u32 sts_boot_dev_sts1_reg,u32 boot_err0_reg,u32 boot_err1_reg)741 int hl_fw_cpucp_handshake(struct hl_device *hdev,
742 u32 sts_boot_dev_sts0_reg,
743 u32 sts_boot_dev_sts1_reg, u32 boot_err0_reg,
744 u32 boot_err1_reg)
745 {
746 int rc;
747
748 rc = hl_fw_cpucp_info_get(hdev, sts_boot_dev_sts0_reg,
749 sts_boot_dev_sts1_reg, boot_err0_reg,
750 boot_err1_reg);
751 if (rc)
752 return rc;
753
754 return hl_fw_send_msi_info_msg(hdev);
755 }
756
hl_fw_get_eeprom_data(struct hl_device * hdev,void * data,size_t max_size)757 int hl_fw_get_eeprom_data(struct hl_device *hdev, void *data, size_t max_size)
758 {
759 struct cpucp_packet pkt = {};
760 void *eeprom_info_cpu_addr;
761 dma_addr_t eeprom_info_dma_addr;
762 u64 result;
763 int rc;
764
765 eeprom_info_cpu_addr =
766 hdev->asic_funcs->cpu_accessible_dma_pool_alloc(hdev,
767 max_size, &eeprom_info_dma_addr);
768 if (!eeprom_info_cpu_addr) {
769 dev_err(hdev->dev,
770 "Failed to allocate DMA memory for CPU-CP EEPROM packet\n");
771 return -ENOMEM;
772 }
773
774 memset(eeprom_info_cpu_addr, 0, max_size);
775
776 pkt.ctl = cpu_to_le32(CPUCP_PACKET_EEPROM_DATA_GET <<
777 CPUCP_PKT_CTL_OPCODE_SHIFT);
778 pkt.addr = cpu_to_le64(eeprom_info_dma_addr);
779 pkt.data_max_size = cpu_to_le32(max_size);
780
781 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
782 HL_CPUCP_EEPROM_TIMEOUT_USEC, &result);
783
784 if (rc) {
785 dev_err(hdev->dev,
786 "Failed to handle CPU-CP EEPROM packet, error %d\n",
787 rc);
788 goto out;
789 }
790
791 /* result contains the actual size */
792 memcpy(data, eeprom_info_cpu_addr, min((size_t)result, max_size));
793
794 out:
795 hdev->asic_funcs->cpu_accessible_dma_pool_free(hdev, max_size,
796 eeprom_info_cpu_addr);
797
798 return rc;
799 }
800
hl_fw_cpucp_pci_counters_get(struct hl_device * hdev,struct hl_info_pci_counters * counters)801 int hl_fw_cpucp_pci_counters_get(struct hl_device *hdev,
802 struct hl_info_pci_counters *counters)
803 {
804 struct cpucp_packet pkt = {};
805 u64 result;
806 int rc;
807
808 pkt.ctl = cpu_to_le32(CPUCP_PACKET_PCIE_THROUGHPUT_GET <<
809 CPUCP_PKT_CTL_OPCODE_SHIFT);
810
811 /* Fetch PCI rx counter */
812 pkt.index = cpu_to_le32(cpucp_pcie_throughput_rx);
813 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
814 HL_CPUCP_INFO_TIMEOUT_USEC, &result);
815 if (rc) {
816 dev_err(hdev->dev,
817 "Failed to handle CPU-CP PCI info pkt, error %d\n", rc);
818 return rc;
819 }
820 counters->rx_throughput = result;
821
822 memset(&pkt, 0, sizeof(pkt));
823 pkt.ctl = cpu_to_le32(CPUCP_PACKET_PCIE_THROUGHPUT_GET <<
824 CPUCP_PKT_CTL_OPCODE_SHIFT);
825
826 /* Fetch PCI tx counter */
827 pkt.index = cpu_to_le32(cpucp_pcie_throughput_tx);
828 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
829 HL_CPUCP_INFO_TIMEOUT_USEC, &result);
830 if (rc) {
831 dev_err(hdev->dev,
832 "Failed to handle CPU-CP PCI info pkt, error %d\n", rc);
833 return rc;
834 }
835 counters->tx_throughput = result;
836
837 /* Fetch PCI replay counter */
838 memset(&pkt, 0, sizeof(pkt));
839 pkt.ctl = cpu_to_le32(CPUCP_PACKET_PCIE_REPLAY_CNT_GET <<
840 CPUCP_PKT_CTL_OPCODE_SHIFT);
841
842 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
843 HL_CPUCP_INFO_TIMEOUT_USEC, &result);
844 if (rc) {
845 dev_err(hdev->dev,
846 "Failed to handle CPU-CP PCI info pkt, error %d\n", rc);
847 return rc;
848 }
849 counters->replay_cnt = (u32) result;
850
851 return rc;
852 }
853
hl_fw_cpucp_total_energy_get(struct hl_device * hdev,u64 * total_energy)854 int hl_fw_cpucp_total_energy_get(struct hl_device *hdev, u64 *total_energy)
855 {
856 struct cpucp_packet pkt = {};
857 u64 result;
858 int rc;
859
860 pkt.ctl = cpu_to_le32(CPUCP_PACKET_TOTAL_ENERGY_GET <<
861 CPUCP_PKT_CTL_OPCODE_SHIFT);
862
863 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
864 HL_CPUCP_INFO_TIMEOUT_USEC, &result);
865 if (rc) {
866 dev_err(hdev->dev,
867 "Failed to handle CpuCP total energy pkt, error %d\n",
868 rc);
869 return rc;
870 }
871
872 *total_energy = result;
873
874 return rc;
875 }
876
get_used_pll_index(struct hl_device * hdev,u32 input_pll_index,enum pll_index * pll_index)877 int get_used_pll_index(struct hl_device *hdev, u32 input_pll_index,
878 enum pll_index *pll_index)
879 {
880 struct asic_fixed_properties *prop = &hdev->asic_prop;
881 u8 pll_byte, pll_bit_off;
882 bool dynamic_pll;
883 int fw_pll_idx;
884
885 dynamic_pll = !!(prop->fw_app_cpu_boot_dev_sts0 &
886 CPU_BOOT_DEV_STS0_DYN_PLL_EN);
887
888 if (!dynamic_pll) {
889 /*
890 * in case we are working with legacy FW (each asic has unique
891 * PLL numbering) use the driver based index as they are
892 * aligned with fw legacy numbering
893 */
894 *pll_index = input_pll_index;
895 return 0;
896 }
897
898 /* retrieve a FW compatible PLL index based on
899 * ASIC specific user request
900 */
901 fw_pll_idx = hdev->asic_funcs->map_pll_idx_to_fw_idx(input_pll_index);
902 if (fw_pll_idx < 0) {
903 dev_err(hdev->dev, "Invalid PLL index (%u) error %d\n",
904 input_pll_index, fw_pll_idx);
905 return -EINVAL;
906 }
907
908 /* PLL map is a u8 array */
909 pll_byte = prop->cpucp_info.pll_map[fw_pll_idx >> 3];
910 pll_bit_off = fw_pll_idx & 0x7;
911
912 if (!(pll_byte & BIT(pll_bit_off))) {
913 dev_err(hdev->dev, "PLL index %d is not supported\n",
914 fw_pll_idx);
915 return -EINVAL;
916 }
917
918 *pll_index = fw_pll_idx;
919
920 return 0;
921 }
922
hl_fw_cpucp_pll_info_get(struct hl_device * hdev,u32 pll_index,u16 * pll_freq_arr)923 int hl_fw_cpucp_pll_info_get(struct hl_device *hdev, u32 pll_index,
924 u16 *pll_freq_arr)
925 {
926 struct cpucp_packet pkt;
927 enum pll_index used_pll_idx;
928 u64 result;
929 int rc;
930
931 rc = get_used_pll_index(hdev, pll_index, &used_pll_idx);
932 if (rc)
933 return rc;
934
935 memset(&pkt, 0, sizeof(pkt));
936
937 pkt.ctl = cpu_to_le32(CPUCP_PACKET_PLL_INFO_GET <<
938 CPUCP_PKT_CTL_OPCODE_SHIFT);
939 pkt.pll_type = __cpu_to_le16((u16)used_pll_idx);
940
941 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
942 HL_CPUCP_INFO_TIMEOUT_USEC, &result);
943 if (rc)
944 dev_err(hdev->dev, "Failed to read PLL info, error %d\n", rc);
945
946 pll_freq_arr[0] = FIELD_GET(CPUCP_PKT_RES_PLL_OUT0_MASK, result);
947 pll_freq_arr[1] = FIELD_GET(CPUCP_PKT_RES_PLL_OUT1_MASK, result);
948 pll_freq_arr[2] = FIELD_GET(CPUCP_PKT_RES_PLL_OUT2_MASK, result);
949 pll_freq_arr[3] = FIELD_GET(CPUCP_PKT_RES_PLL_OUT3_MASK, result);
950
951 return rc;
952 }
953
hl_fw_cpucp_power_get(struct hl_device * hdev,u64 * power)954 int hl_fw_cpucp_power_get(struct hl_device *hdev, u64 *power)
955 {
956 struct cpucp_packet pkt;
957 u64 result;
958 int rc;
959
960 memset(&pkt, 0, sizeof(pkt));
961
962 pkt.ctl = cpu_to_le32(CPUCP_PACKET_POWER_GET <<
963 CPUCP_PKT_CTL_OPCODE_SHIFT);
964
965 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
966 HL_CPUCP_INFO_TIMEOUT_USEC, &result);
967 if (rc) {
968 dev_err(hdev->dev, "Failed to read power, error %d\n", rc);
969 return rc;
970 }
971
972 *power = result;
973
974 return rc;
975 }
976
hl_fw_ask_hard_reset_without_linux(struct hl_device * hdev)977 void hl_fw_ask_hard_reset_without_linux(struct hl_device *hdev)
978 {
979 struct static_fw_load_mgr *static_loader =
980 &hdev->fw_loader.static_loader;
981 int rc;
982
983 if (hdev->asic_prop.dynamic_fw_load) {
984 rc = hl_fw_dynamic_send_protocol_cmd(hdev, &hdev->fw_loader,
985 COMMS_RST_DEV, 0, false,
986 hdev->fw_loader.cpu_timeout);
987 if (rc)
988 dev_warn(hdev->dev, "Failed sending COMMS_RST_DEV\n");
989 } else {
990 WREG32(static_loader->kmd_msg_to_cpu_reg, KMD_MSG_RST_DEV);
991 }
992 }
993
hl_fw_ask_halt_machine_without_linux(struct hl_device * hdev)994 void hl_fw_ask_halt_machine_without_linux(struct hl_device *hdev)
995 {
996 struct static_fw_load_mgr *static_loader =
997 &hdev->fw_loader.static_loader;
998 int rc;
999
1000 if (hdev->device_cpu_is_halted)
1001 return;
1002
1003 /* Stop device CPU to make sure nothing bad happens */
1004 if (hdev->asic_prop.dynamic_fw_load) {
1005 rc = hl_fw_dynamic_send_protocol_cmd(hdev, &hdev->fw_loader,
1006 COMMS_GOTO_WFE, 0, true,
1007 hdev->fw_loader.cpu_timeout);
1008 if (rc)
1009 dev_warn(hdev->dev, "Failed sending COMMS_GOTO_WFE\n");
1010 } else {
1011 WREG32(static_loader->kmd_msg_to_cpu_reg, KMD_MSG_GOTO_WFE);
1012 msleep(static_loader->cpu_reset_wait_msec);
1013
1014 /* Must clear this register in order to prevent preboot
1015 * from reading WFE after reboot
1016 */
1017 WREG32(static_loader->kmd_msg_to_cpu_reg, KMD_MSG_NA);
1018 }
1019
1020 hdev->device_cpu_is_halted = true;
1021 }
1022
detect_cpu_boot_status(struct hl_device * hdev,u32 status)1023 static void detect_cpu_boot_status(struct hl_device *hdev, u32 status)
1024 {
1025 /* Some of the status codes below are deprecated in newer f/w
1026 * versions but we keep them here for backward compatibility
1027 */
1028 switch (status) {
1029 case CPU_BOOT_STATUS_NA:
1030 dev_err(hdev->dev,
1031 "Device boot progress - BTL did NOT run\n");
1032 break;
1033 case CPU_BOOT_STATUS_IN_WFE:
1034 dev_err(hdev->dev,
1035 "Device boot progress - Stuck inside WFE loop\n");
1036 break;
1037 case CPU_BOOT_STATUS_IN_BTL:
1038 dev_err(hdev->dev,
1039 "Device boot progress - Stuck in BTL\n");
1040 break;
1041 case CPU_BOOT_STATUS_IN_PREBOOT:
1042 dev_err(hdev->dev,
1043 "Device boot progress - Stuck in Preboot\n");
1044 break;
1045 case CPU_BOOT_STATUS_IN_SPL:
1046 dev_err(hdev->dev,
1047 "Device boot progress - Stuck in SPL\n");
1048 break;
1049 case CPU_BOOT_STATUS_IN_UBOOT:
1050 dev_err(hdev->dev,
1051 "Device boot progress - Stuck in u-boot\n");
1052 break;
1053 case CPU_BOOT_STATUS_DRAM_INIT_FAIL:
1054 dev_err(hdev->dev,
1055 "Device boot progress - DRAM initialization failed\n");
1056 break;
1057 case CPU_BOOT_STATUS_UBOOT_NOT_READY:
1058 dev_err(hdev->dev,
1059 "Device boot progress - Cannot boot\n");
1060 break;
1061 case CPU_BOOT_STATUS_TS_INIT_FAIL:
1062 dev_err(hdev->dev,
1063 "Device boot progress - Thermal Sensor initialization failed\n");
1064 break;
1065 case CPU_BOOT_STATUS_SECURITY_READY:
1066 dev_err(hdev->dev,
1067 "Device boot progress - Stuck in preboot after security initialization\n");
1068 break;
1069 default:
1070 dev_err(hdev->dev,
1071 "Device boot progress - Invalid status code %d\n",
1072 status);
1073 break;
1074 }
1075 }
1076
hl_fw_read_preboot_caps(struct hl_device * hdev,u32 cpu_boot_status_reg,u32 sts_boot_dev_sts0_reg,u32 sts_boot_dev_sts1_reg,u32 boot_err0_reg,u32 boot_err1_reg,u32 timeout)1077 static int hl_fw_read_preboot_caps(struct hl_device *hdev,
1078 u32 cpu_boot_status_reg,
1079 u32 sts_boot_dev_sts0_reg,
1080 u32 sts_boot_dev_sts1_reg,
1081 u32 boot_err0_reg, u32 boot_err1_reg,
1082 u32 timeout)
1083 {
1084 struct asic_fixed_properties *prop = &hdev->asic_prop;
1085 u32 status, reg_val;
1086 int rc;
1087
1088 /* Need to check two possible scenarios:
1089 *
1090 * CPU_BOOT_STATUS_WAITING_FOR_BOOT_FIT - for newer firmwares where
1091 * the preboot is waiting for the boot fit
1092 *
1093 * All other status values - for older firmwares where the uboot was
1094 * loaded from the FLASH
1095 */
1096 rc = hl_poll_timeout(
1097 hdev,
1098 cpu_boot_status_reg,
1099 status,
1100 (status == CPU_BOOT_STATUS_IN_UBOOT) ||
1101 (status == CPU_BOOT_STATUS_DRAM_RDY) ||
1102 (status == CPU_BOOT_STATUS_NIC_FW_RDY) ||
1103 (status == CPU_BOOT_STATUS_READY_TO_BOOT) ||
1104 (status == CPU_BOOT_STATUS_SRAM_AVAIL) ||
1105 (status == CPU_BOOT_STATUS_WAITING_FOR_BOOT_FIT),
1106 FW_CPU_STATUS_POLL_INTERVAL_USEC,
1107 timeout);
1108
1109 if (rc) {
1110 dev_err(hdev->dev, "CPU boot ready status timeout\n");
1111 detect_cpu_boot_status(hdev, status);
1112
1113 /* If we read all FF, then something is totally wrong, no point
1114 * of reading specific errors
1115 */
1116 if (status != -1)
1117 fw_read_errors(hdev, boot_err0_reg, boot_err1_reg,
1118 sts_boot_dev_sts0_reg,
1119 sts_boot_dev_sts1_reg);
1120 return -EIO;
1121 }
1122
1123 /*
1124 * the registers DEV_STS* contain FW capabilities/features.
1125 * We can rely on this registers only if bit CPU_BOOT_DEV_STS*_ENABLED
1126 * is set.
1127 * In the first read of this register we store the value of this
1128 * register ONLY if the register is enabled (which will be propagated
1129 * to next stages) and also mark the register as valid.
1130 * In case it is not enabled the stored value will be left 0- all
1131 * caps/features are off
1132 */
1133 reg_val = RREG32(sts_boot_dev_sts0_reg);
1134 if (reg_val & CPU_BOOT_DEV_STS0_ENABLED) {
1135 prop->fw_cpu_boot_dev_sts0_valid = true;
1136 prop->fw_preboot_cpu_boot_dev_sts0 = reg_val;
1137 }
1138
1139 reg_val = RREG32(sts_boot_dev_sts1_reg);
1140 if (reg_val & CPU_BOOT_DEV_STS1_ENABLED) {
1141 prop->fw_cpu_boot_dev_sts1_valid = true;
1142 prop->fw_preboot_cpu_boot_dev_sts1 = reg_val;
1143 }
1144
1145 prop->dynamic_fw_load = !!(prop->fw_preboot_cpu_boot_dev_sts0 &
1146 CPU_BOOT_DEV_STS0_FW_LD_COM_EN);
1147
1148 /* initialize FW loader once we know what load protocol is used */
1149 hdev->asic_funcs->init_firmware_loader(hdev);
1150
1151 dev_dbg(hdev->dev, "Attempting %s FW load\n",
1152 prop->dynamic_fw_load ? "dynamic" : "legacy");
1153 return 0;
1154 }
1155
hl_fw_static_read_device_fw_version(struct hl_device * hdev,enum hl_fw_component fwc)1156 static int hl_fw_static_read_device_fw_version(struct hl_device *hdev,
1157 enum hl_fw_component fwc)
1158 {
1159 struct asic_fixed_properties *prop = &hdev->asic_prop;
1160 struct fw_load_mgr *fw_loader = &hdev->fw_loader;
1161 struct static_fw_load_mgr *static_loader;
1162 char *dest, *boot_ver, *preboot_ver;
1163 u32 ver_off, limit;
1164 const char *name;
1165 char btl_ver[32];
1166
1167 static_loader = &hdev->fw_loader.static_loader;
1168
1169 switch (fwc) {
1170 case FW_COMP_BOOT_FIT:
1171 ver_off = RREG32(static_loader->boot_fit_version_offset_reg);
1172 dest = prop->uboot_ver;
1173 name = "Boot-fit";
1174 limit = static_loader->boot_fit_version_max_off;
1175 break;
1176 case FW_COMP_PREBOOT:
1177 ver_off = RREG32(static_loader->preboot_version_offset_reg);
1178 dest = prop->preboot_ver;
1179 name = "Preboot";
1180 limit = static_loader->preboot_version_max_off;
1181 break;
1182 default:
1183 dev_warn(hdev->dev, "Undefined FW component: %d\n", fwc);
1184 return -EIO;
1185 }
1186
1187 ver_off &= static_loader->sram_offset_mask;
1188
1189 if (ver_off < limit) {
1190 memcpy_fromio(dest,
1191 hdev->pcie_bar[fw_loader->sram_bar_id] + ver_off,
1192 VERSION_MAX_LEN);
1193 } else {
1194 dev_err(hdev->dev, "%s version offset (0x%x) is above SRAM\n",
1195 name, ver_off);
1196 strscpy(dest, "unavailable", VERSION_MAX_LEN);
1197 return -EIO;
1198 }
1199
1200 if (fwc == FW_COMP_BOOT_FIT) {
1201 boot_ver = extract_fw_ver_from_str(prop->uboot_ver);
1202 if (boot_ver) {
1203 dev_info(hdev->dev, "boot-fit version %s\n", boot_ver);
1204 kfree(boot_ver);
1205 }
1206 } else if (fwc == FW_COMP_PREBOOT) {
1207 preboot_ver = strnstr(prop->preboot_ver, "Preboot",
1208 VERSION_MAX_LEN);
1209 if (preboot_ver && preboot_ver != prop->preboot_ver) {
1210 strscpy(btl_ver, prop->preboot_ver,
1211 min((int) (preboot_ver - prop->preboot_ver),
1212 31));
1213 dev_info(hdev->dev, "%s\n", btl_ver);
1214 }
1215
1216 preboot_ver = extract_fw_ver_from_str(prop->preboot_ver);
1217 if (preboot_ver) {
1218 dev_info(hdev->dev, "preboot version %s\n",
1219 preboot_ver);
1220 kfree(preboot_ver);
1221 }
1222 }
1223
1224 return 0;
1225 }
1226
1227 /**
1228 * hl_fw_preboot_update_state - update internal data structures during
1229 * handshake with preboot
1230 *
1231 *
1232 * @hdev: pointer to the habanalabs device structure
1233 *
1234 * @return 0 on success, otherwise non-zero error code
1235 */
hl_fw_preboot_update_state(struct hl_device * hdev)1236 static void hl_fw_preboot_update_state(struct hl_device *hdev)
1237 {
1238 struct asic_fixed_properties *prop = &hdev->asic_prop;
1239 u32 cpu_boot_dev_sts0, cpu_boot_dev_sts1;
1240
1241 cpu_boot_dev_sts0 = prop->fw_preboot_cpu_boot_dev_sts0;
1242 cpu_boot_dev_sts1 = prop->fw_preboot_cpu_boot_dev_sts1;
1243
1244 /* We read boot_dev_sts registers multiple times during boot:
1245 * 1. preboot - a. Check whether the security status bits are valid
1246 * b. Check whether fw security is enabled
1247 * c. Check whether hard reset is done by preboot
1248 * 2. boot cpu - a. Fetch boot cpu security status
1249 * b. Check whether hard reset is done by boot cpu
1250 * 3. FW application - a. Fetch fw application security status
1251 * b. Check whether hard reset is done by fw app
1252 */
1253 prop->hard_reset_done_by_fw =
1254 !!(cpu_boot_dev_sts0 & CPU_BOOT_DEV_STS0_FW_HARD_RST_EN);
1255
1256 dev_dbg(hdev->dev, "Firmware preboot boot device status0 %#x\n",
1257 cpu_boot_dev_sts0);
1258
1259 dev_dbg(hdev->dev, "Firmware preboot boot device status1 %#x\n",
1260 cpu_boot_dev_sts1);
1261
1262 dev_dbg(hdev->dev, "Firmware preboot hard-reset is %s\n",
1263 prop->hard_reset_done_by_fw ? "enabled" : "disabled");
1264
1265 dev_dbg(hdev->dev, "firmware-level security is %s\n",
1266 prop->fw_security_enabled ? "enabled" : "disabled");
1267
1268 dev_dbg(hdev->dev, "GIC controller is %s\n",
1269 prop->gic_interrupts_enable ? "enabled" : "disabled");
1270 }
1271
hl_fw_static_read_preboot_status(struct hl_device * hdev)1272 static int hl_fw_static_read_preboot_status(struct hl_device *hdev)
1273 {
1274 int rc;
1275
1276 rc = hl_fw_static_read_device_fw_version(hdev, FW_COMP_PREBOOT);
1277 if (rc)
1278 return rc;
1279
1280 return 0;
1281 }
1282
hl_fw_read_preboot_status(struct hl_device * hdev,u32 cpu_boot_status_reg,u32 sts_boot_dev_sts0_reg,u32 sts_boot_dev_sts1_reg,u32 boot_err0_reg,u32 boot_err1_reg,u32 timeout)1283 int hl_fw_read_preboot_status(struct hl_device *hdev, u32 cpu_boot_status_reg,
1284 u32 sts_boot_dev_sts0_reg,
1285 u32 sts_boot_dev_sts1_reg, u32 boot_err0_reg,
1286 u32 boot_err1_reg, u32 timeout)
1287 {
1288 int rc;
1289
1290 /* pldm was added for cases in which we use preboot on pldm and want
1291 * to load boot fit, but we can't wait for preboot because it runs
1292 * very slowly
1293 */
1294 if (!(hdev->fw_components & FW_TYPE_PREBOOT_CPU) || hdev->pldm)
1295 return 0;
1296
1297 /*
1298 * In order to determine boot method (static VS dymanic) we need to
1299 * read the boot caps register
1300 */
1301 rc = hl_fw_read_preboot_caps(hdev, cpu_boot_status_reg,
1302 sts_boot_dev_sts0_reg,
1303 sts_boot_dev_sts1_reg, boot_err0_reg,
1304 boot_err1_reg, timeout);
1305 if (rc)
1306 return rc;
1307
1308 hl_fw_preboot_update_state(hdev);
1309
1310 /* no need to read preboot status in dynamic load */
1311 if (hdev->asic_prop.dynamic_fw_load)
1312 return 0;
1313
1314 return hl_fw_static_read_preboot_status(hdev);
1315 }
1316
1317 /* associate string with COMM status */
1318 static char *hl_dynamic_fw_status_str[COMMS_STS_INVLD_LAST] = {
1319 [COMMS_STS_NOOP] = "NOOP",
1320 [COMMS_STS_ACK] = "ACK",
1321 [COMMS_STS_OK] = "OK",
1322 [COMMS_STS_ERR] = "ERR",
1323 [COMMS_STS_VALID_ERR] = "VALID_ERR",
1324 [COMMS_STS_TIMEOUT_ERR] = "TIMEOUT_ERR",
1325 };
1326
1327 /**
1328 * hl_fw_dynamic_report_error_status - report error status
1329 *
1330 * @hdev: pointer to the habanalabs device structure
1331 * @status: value of FW status register
1332 * @expected_status: the expected status
1333 */
hl_fw_dynamic_report_error_status(struct hl_device * hdev,u32 status,enum comms_sts expected_status)1334 static void hl_fw_dynamic_report_error_status(struct hl_device *hdev,
1335 u32 status,
1336 enum comms_sts expected_status)
1337 {
1338 enum comms_sts comm_status =
1339 FIELD_GET(COMMS_STATUS_STATUS_MASK, status);
1340
1341 if (comm_status < COMMS_STS_INVLD_LAST)
1342 dev_err(hdev->dev, "Device status %s, expected status: %s\n",
1343 hl_dynamic_fw_status_str[comm_status],
1344 hl_dynamic_fw_status_str[expected_status]);
1345 else
1346 dev_err(hdev->dev, "Device status unknown %d, expected status: %s\n",
1347 comm_status,
1348 hl_dynamic_fw_status_str[expected_status]);
1349 }
1350
1351 /**
1352 * hl_fw_dynamic_send_cmd - send LKD to FW cmd
1353 *
1354 * @hdev: pointer to the habanalabs device structure
1355 * @fw_loader: managing structure for loading device's FW
1356 * @cmd: LKD to FW cmd code
1357 * @size: size of next FW component to be loaded (0 if not necessary)
1358 *
1359 * LDK to FW exact command layout is defined at struct comms_command.
1360 * note: the size argument is used only when the next FW component should be
1361 * loaded, otherwise it shall be 0. the size is used by the FW in later
1362 * protocol stages and when sending only indicating the amount of memory
1363 * to be allocated by the FW to receive the next boot component.
1364 */
hl_fw_dynamic_send_cmd(struct hl_device * hdev,struct fw_load_mgr * fw_loader,enum comms_cmd cmd,unsigned int size)1365 static void hl_fw_dynamic_send_cmd(struct hl_device *hdev,
1366 struct fw_load_mgr *fw_loader,
1367 enum comms_cmd cmd, unsigned int size)
1368 {
1369 struct cpu_dyn_regs *dyn_regs;
1370 u32 val;
1371
1372 dyn_regs = &fw_loader->dynamic_loader.comm_desc.cpu_dyn_regs;
1373
1374 val = FIELD_PREP(COMMS_COMMAND_CMD_MASK, cmd);
1375 val |= FIELD_PREP(COMMS_COMMAND_SIZE_MASK, size);
1376
1377 WREG32(le32_to_cpu(dyn_regs->kmd_msg_to_cpu), val);
1378 }
1379
1380 /**
1381 * hl_fw_dynamic_extract_fw_response - update the FW response
1382 *
1383 * @hdev: pointer to the habanalabs device structure
1384 * @fw_loader: managing structure for loading device's FW
1385 * @response: FW response
1386 * @status: the status read from CPU status register
1387 *
1388 * @return 0 on success, otherwise non-zero error code
1389 */
hl_fw_dynamic_extract_fw_response(struct hl_device * hdev,struct fw_load_mgr * fw_loader,struct fw_response * response,u32 status)1390 static int hl_fw_dynamic_extract_fw_response(struct hl_device *hdev,
1391 struct fw_load_mgr *fw_loader,
1392 struct fw_response *response,
1393 u32 status)
1394 {
1395 response->status = FIELD_GET(COMMS_STATUS_STATUS_MASK, status);
1396 response->ram_offset = FIELD_GET(COMMS_STATUS_OFFSET_MASK, status) <<
1397 COMMS_STATUS_OFFSET_ALIGN_SHIFT;
1398 response->ram_type = FIELD_GET(COMMS_STATUS_RAM_TYPE_MASK, status);
1399
1400 if ((response->ram_type != COMMS_SRAM) &&
1401 (response->ram_type != COMMS_DRAM)) {
1402 dev_err(hdev->dev, "FW status: invalid RAM type %u\n",
1403 response->ram_type);
1404 return -EIO;
1405 }
1406
1407 return 0;
1408 }
1409
1410 /**
1411 * hl_fw_dynamic_wait_for_status - wait for status in dynamic FW load
1412 *
1413 * @hdev: pointer to the habanalabs device structure
1414 * @fw_loader: managing structure for loading device's FW
1415 * @expected_status: expected status to wait for
1416 * @timeout: timeout for status wait
1417 *
1418 * @return 0 on success, otherwise non-zero error code
1419 *
1420 * waiting for status from FW include polling the FW status register until
1421 * expected status is received or timeout occurs (whatever occurs first).
1422 */
hl_fw_dynamic_wait_for_status(struct hl_device * hdev,struct fw_load_mgr * fw_loader,enum comms_sts expected_status,u32 timeout)1423 static int hl_fw_dynamic_wait_for_status(struct hl_device *hdev,
1424 struct fw_load_mgr *fw_loader,
1425 enum comms_sts expected_status,
1426 u32 timeout)
1427 {
1428 struct cpu_dyn_regs *dyn_regs;
1429 u32 status;
1430 int rc;
1431
1432 dyn_regs = &fw_loader->dynamic_loader.comm_desc.cpu_dyn_regs;
1433
1434 /* Wait for expected status */
1435 rc = hl_poll_timeout(
1436 hdev,
1437 le32_to_cpu(dyn_regs->cpu_cmd_status_to_host),
1438 status,
1439 FIELD_GET(COMMS_STATUS_STATUS_MASK, status) == expected_status,
1440 FW_CPU_STATUS_POLL_INTERVAL_USEC,
1441 timeout);
1442
1443 if (rc) {
1444 hl_fw_dynamic_report_error_status(hdev, status,
1445 expected_status);
1446 return -EIO;
1447 }
1448
1449 /*
1450 * skip storing FW response for NOOP to preserve the actual desired
1451 * FW status
1452 */
1453 if (expected_status == COMMS_STS_NOOP)
1454 return 0;
1455
1456 rc = hl_fw_dynamic_extract_fw_response(hdev, fw_loader,
1457 &fw_loader->dynamic_loader.response,
1458 status);
1459 return rc;
1460 }
1461
1462 /**
1463 * hl_fw_dynamic_send_clear_cmd - send clear command to FW
1464 *
1465 * @hdev: pointer to the habanalabs device structure
1466 * @fw_loader: managing structure for loading device's FW
1467 *
1468 * @return 0 on success, otherwise non-zero error code
1469 *
1470 * after command cycle between LKD to FW CPU (i.e. LKD got an expected status
1471 * from FW) we need to clear the CPU status register in order to avoid garbage
1472 * between command cycles.
1473 * This is done by sending clear command and polling the CPU to LKD status
1474 * register to hold the status NOOP
1475 */
hl_fw_dynamic_send_clear_cmd(struct hl_device * hdev,struct fw_load_mgr * fw_loader)1476 static int hl_fw_dynamic_send_clear_cmd(struct hl_device *hdev,
1477 struct fw_load_mgr *fw_loader)
1478 {
1479 hl_fw_dynamic_send_cmd(hdev, fw_loader, COMMS_CLR_STS, 0);
1480
1481 return hl_fw_dynamic_wait_for_status(hdev, fw_loader, COMMS_STS_NOOP,
1482 fw_loader->cpu_timeout);
1483 }
1484
1485 /**
1486 * hl_fw_dynamic_send_protocol_cmd - send LKD to FW cmd and wait for ACK
1487 *
1488 * @hdev: pointer to the habanalabs device structure
1489 * @fw_loader: managing structure for loading device's FW
1490 * @cmd: LKD to FW cmd code
1491 * @size: size of next FW component to be loaded (0 if not necessary)
1492 * @wait_ok: if true also wait for OK response from FW
1493 * @timeout: timeout for status wait
1494 *
1495 * @return 0 on success, otherwise non-zero error code
1496 *
1497 * brief:
1498 * when sending protocol command we have the following steps:
1499 * - send clear (clear command and verify clear status register)
1500 * - send the actual protocol command
1501 * - wait for ACK on the protocol command
1502 * - send clear
1503 * - send NOOP
1504 * if, in addition, the specific protocol command should wait for OK then:
1505 * - wait for OK
1506 * - send clear
1507 * - send NOOP
1508 *
1509 * NOTES:
1510 * send clear: this is necessary in order to clear the status register to avoid
1511 * leftovers between command
1512 * NOOP command: necessary to avoid loop on the clear command by the FW
1513 */
hl_fw_dynamic_send_protocol_cmd(struct hl_device * hdev,struct fw_load_mgr * fw_loader,enum comms_cmd cmd,unsigned int size,bool wait_ok,u32 timeout)1514 int hl_fw_dynamic_send_protocol_cmd(struct hl_device *hdev,
1515 struct fw_load_mgr *fw_loader,
1516 enum comms_cmd cmd, unsigned int size,
1517 bool wait_ok, u32 timeout)
1518 {
1519 int rc;
1520
1521 /* first send clear command to clean former commands */
1522 rc = hl_fw_dynamic_send_clear_cmd(hdev, fw_loader);
1523
1524 /* send the actual command */
1525 hl_fw_dynamic_send_cmd(hdev, fw_loader, cmd, size);
1526
1527 /* wait for ACK for the command */
1528 rc = hl_fw_dynamic_wait_for_status(hdev, fw_loader, COMMS_STS_ACK,
1529 timeout);
1530 if (rc)
1531 return rc;
1532
1533 /* clear command to prepare for NOOP command */
1534 rc = hl_fw_dynamic_send_clear_cmd(hdev, fw_loader);
1535 if (rc)
1536 return rc;
1537
1538 /* send the actual NOOP command */
1539 hl_fw_dynamic_send_cmd(hdev, fw_loader, COMMS_NOOP, 0);
1540
1541 if (!wait_ok)
1542 return 0;
1543
1544 rc = hl_fw_dynamic_wait_for_status(hdev, fw_loader, COMMS_STS_OK,
1545 timeout);
1546 if (rc)
1547 return rc;
1548
1549 /* clear command to prepare for NOOP command */
1550 rc = hl_fw_dynamic_send_clear_cmd(hdev, fw_loader);
1551 if (rc)
1552 return rc;
1553
1554 /* send the actual NOOP command */
1555 hl_fw_dynamic_send_cmd(hdev, fw_loader, COMMS_NOOP, 0);
1556
1557 return 0;
1558 }
1559
1560 /**
1561 * hl_fw_compat_crc32 - CRC compatible with FW
1562 *
1563 * @data: pointer to the data
1564 * @size: size of the data
1565 *
1566 * @return the CRC32 result
1567 *
1568 * NOTE: kernel's CRC32 differ's from standard CRC32 calculation.
1569 * in order to be aligned we need to flip the bits of both the input
1570 * initial CRC and kernel's CRC32 result.
1571 * in addition both sides use initial CRC of 0,
1572 */
hl_fw_compat_crc32(u8 * data,size_t size)1573 static u32 hl_fw_compat_crc32(u8 *data, size_t size)
1574 {
1575 return ~crc32_le(~((u32)0), data, size);
1576 }
1577
1578 /**
1579 * hl_fw_dynamic_validate_memory_bound - validate memory bounds for memory
1580 * transfer (image or descriptor) between
1581 * host and FW
1582 *
1583 * @hdev: pointer to the habanalabs device structure
1584 * @addr: device address of memory transfer
1585 * @size: memory transter size
1586 * @region: PCI memory region
1587 *
1588 * @return 0 on success, otherwise non-zero error code
1589 */
hl_fw_dynamic_validate_memory_bound(struct hl_device * hdev,u64 addr,size_t size,struct pci_mem_region * region)1590 static int hl_fw_dynamic_validate_memory_bound(struct hl_device *hdev,
1591 u64 addr, size_t size,
1592 struct pci_mem_region *region)
1593 {
1594 u64 end_addr;
1595
1596 /* now make sure that the memory transfer is within region's bounds */
1597 end_addr = addr + size;
1598 if (end_addr >= region->region_base + region->region_size) {
1599 dev_err(hdev->dev,
1600 "dynamic FW load: memory transfer end address out of memory region bounds. addr: %llx\n",
1601 end_addr);
1602 return -EIO;
1603 }
1604
1605 /*
1606 * now make sure memory transfer is within predefined BAR bounds.
1607 * this is to make sure we do not need to set the bar (e.g. for DRAM
1608 * memory transfers)
1609 */
1610 if (end_addr >= region->region_base - region->offset_in_bar +
1611 region->bar_size) {
1612 dev_err(hdev->dev,
1613 "FW image beyond PCI BAR bounds\n");
1614 return -EIO;
1615 }
1616
1617 return 0;
1618 }
1619
1620 /**
1621 * hl_fw_dynamic_validate_descriptor - validate FW descriptor
1622 *
1623 * @hdev: pointer to the habanalabs device structure
1624 * @fw_loader: managing structure for loading device's FW
1625 * @fw_desc: the descriptor form FW
1626 *
1627 * @return 0 on success, otherwise non-zero error code
1628 */
hl_fw_dynamic_validate_descriptor(struct hl_device * hdev,struct fw_load_mgr * fw_loader,struct lkd_fw_comms_desc * fw_desc)1629 static int hl_fw_dynamic_validate_descriptor(struct hl_device *hdev,
1630 struct fw_load_mgr *fw_loader,
1631 struct lkd_fw_comms_desc *fw_desc)
1632 {
1633 struct pci_mem_region *region;
1634 enum pci_region region_id;
1635 size_t data_size;
1636 u32 data_crc32;
1637 u8 *data_ptr;
1638 u64 addr;
1639 int rc;
1640
1641 if (le32_to_cpu(fw_desc->header.magic) != HL_COMMS_DESC_MAGIC) {
1642 dev_err(hdev->dev, "Invalid magic for dynamic FW descriptor (%x)\n",
1643 fw_desc->header.magic);
1644 return -EIO;
1645 }
1646
1647 if (fw_desc->header.version != HL_COMMS_DESC_VER) {
1648 dev_err(hdev->dev, "Invalid version for dynamic FW descriptor (%x)\n",
1649 fw_desc->header.version);
1650 return -EIO;
1651 }
1652
1653 /*
1654 * calc CRC32 of data without header.
1655 * note that no alignment/stride address issues here as all structures
1656 * are 64 bit padded
1657 */
1658 data_size = sizeof(struct lkd_fw_comms_desc) -
1659 sizeof(struct comms_desc_header);
1660 data_ptr = (u8 *)fw_desc + sizeof(struct comms_desc_header);
1661
1662 if (le16_to_cpu(fw_desc->header.size) != data_size) {
1663 dev_err(hdev->dev,
1664 "Invalid descriptor size 0x%x, expected size 0x%zx\n",
1665 le16_to_cpu(fw_desc->header.size), data_size);
1666 return -EIO;
1667 }
1668
1669 data_crc32 = hl_fw_compat_crc32(data_ptr, data_size);
1670
1671 if (data_crc32 != le32_to_cpu(fw_desc->header.crc32)) {
1672 dev_err(hdev->dev,
1673 "CRC32 mismatch for dynamic FW descriptor (%x:%x)\n",
1674 data_crc32, fw_desc->header.crc32);
1675 return -EIO;
1676 }
1677
1678 /* find memory region to which to copy the image */
1679 addr = le64_to_cpu(fw_desc->img_addr);
1680 region_id = hl_get_pci_memory_region(hdev, addr);
1681 if ((region_id != PCI_REGION_SRAM) &&
1682 ((region_id != PCI_REGION_DRAM))) {
1683 dev_err(hdev->dev,
1684 "Invalid region to copy FW image address=%llx\n", addr);
1685 return -EIO;
1686 }
1687
1688 region = &hdev->pci_mem_region[region_id];
1689
1690 /* store the region for the copy stage */
1691 fw_loader->dynamic_loader.image_region = region;
1692
1693 /*
1694 * here we know that the start address is valid, now make sure that the
1695 * image is within region's bounds
1696 */
1697 rc = hl_fw_dynamic_validate_memory_bound(hdev, addr,
1698 fw_loader->dynamic_loader.fw_image_size,
1699 region);
1700 if (rc) {
1701 dev_err(hdev->dev,
1702 "invalid mem transfer request for FW image\n");
1703 return rc;
1704 }
1705
1706 /* here we can mark the descriptor as valid as the content has been validated */
1707 fw_loader->dynamic_loader.fw_desc_valid = true;
1708
1709 return 0;
1710 }
1711
hl_fw_dynamic_validate_response(struct hl_device * hdev,struct fw_response * response,struct pci_mem_region * region)1712 static int hl_fw_dynamic_validate_response(struct hl_device *hdev,
1713 struct fw_response *response,
1714 struct pci_mem_region *region)
1715 {
1716 u64 device_addr;
1717 int rc;
1718
1719 device_addr = region->region_base + response->ram_offset;
1720
1721 /*
1722 * validate that the descriptor is within region's bounds
1723 * Note that as the start address was supplied according to the RAM
1724 * type- testing only the end address is enough
1725 */
1726 rc = hl_fw_dynamic_validate_memory_bound(hdev, device_addr,
1727 sizeof(struct lkd_fw_comms_desc),
1728 region);
1729 return rc;
1730 }
1731
1732 /**
1733 * hl_fw_dynamic_read_and_validate_descriptor - read and validate FW descriptor
1734 *
1735 * @hdev: pointer to the habanalabs device structure
1736 * @fw_loader: managing structure for loading device's FW
1737 *
1738 * @return 0 on success, otherwise non-zero error code
1739 */
hl_fw_dynamic_read_and_validate_descriptor(struct hl_device * hdev,struct fw_load_mgr * fw_loader)1740 static int hl_fw_dynamic_read_and_validate_descriptor(struct hl_device *hdev,
1741 struct fw_load_mgr *fw_loader)
1742 {
1743 struct lkd_fw_comms_desc *fw_desc;
1744 struct pci_mem_region *region;
1745 struct fw_response *response;
1746 enum pci_region region_id;
1747 void __iomem *src;
1748 int rc;
1749
1750 fw_desc = &fw_loader->dynamic_loader.comm_desc;
1751 response = &fw_loader->dynamic_loader.response;
1752
1753 region_id = (response->ram_type == COMMS_SRAM) ?
1754 PCI_REGION_SRAM : PCI_REGION_DRAM;
1755
1756 region = &hdev->pci_mem_region[region_id];
1757
1758 rc = hl_fw_dynamic_validate_response(hdev, response, region);
1759 if (rc) {
1760 dev_err(hdev->dev,
1761 "invalid mem transfer request for FW descriptor\n");
1762 return rc;
1763 }
1764
1765 /*
1766 * extract address to copy the descriptor from
1767 * in addition, as the descriptor value is going to be over-ridden by new data- we mark it
1768 * as invalid.
1769 * it will be marked again as valid once validated
1770 */
1771 fw_loader->dynamic_loader.fw_desc_valid = false;
1772 src = hdev->pcie_bar[region->bar_id] + region->offset_in_bar +
1773 response->ram_offset;
1774 memcpy_fromio(fw_desc, src, sizeof(struct lkd_fw_comms_desc));
1775
1776 return hl_fw_dynamic_validate_descriptor(hdev, fw_loader, fw_desc);
1777 }
1778
1779 /**
1780 * hl_fw_dynamic_request_descriptor - handshake with CPU to get FW descriptor
1781 *
1782 * @hdev: pointer to the habanalabs device structure
1783 * @fw_loader: managing structure for loading device's FW
1784 * @next_image_size: size to allocate for next FW component
1785 *
1786 * @return 0 on success, otherwise non-zero error code
1787 */
hl_fw_dynamic_request_descriptor(struct hl_device * hdev,struct fw_load_mgr * fw_loader,size_t next_image_size)1788 static int hl_fw_dynamic_request_descriptor(struct hl_device *hdev,
1789 struct fw_load_mgr *fw_loader,
1790 size_t next_image_size)
1791 {
1792 int rc;
1793
1794 rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_PREP_DESC,
1795 next_image_size, true,
1796 fw_loader->cpu_timeout);
1797 if (rc)
1798 return rc;
1799
1800 return hl_fw_dynamic_read_and_validate_descriptor(hdev, fw_loader);
1801 }
1802
1803 /**
1804 * hl_fw_dynamic_read_device_fw_version - read FW version to exposed properties
1805 *
1806 * @hdev: pointer to the habanalabs device structure
1807 * @fwc: the firmware component
1808 * @fw_version: fw component's version string
1809 */
hl_fw_dynamic_read_device_fw_version(struct hl_device * hdev,enum hl_fw_component fwc,const char * fw_version)1810 static void hl_fw_dynamic_read_device_fw_version(struct hl_device *hdev,
1811 enum hl_fw_component fwc,
1812 const char *fw_version)
1813 {
1814 struct asic_fixed_properties *prop = &hdev->asic_prop;
1815 char *preboot_ver, *boot_ver;
1816 char btl_ver[32];
1817
1818 switch (fwc) {
1819 case FW_COMP_BOOT_FIT:
1820 strscpy(prop->uboot_ver, fw_version, VERSION_MAX_LEN);
1821 boot_ver = extract_fw_ver_from_str(prop->uboot_ver);
1822 if (boot_ver) {
1823 dev_info(hdev->dev, "boot-fit version %s\n", boot_ver);
1824 kfree(boot_ver);
1825 }
1826
1827 break;
1828 case FW_COMP_PREBOOT:
1829 strscpy(prop->preboot_ver, fw_version, VERSION_MAX_LEN);
1830 preboot_ver = strnstr(prop->preboot_ver, "Preboot",
1831 VERSION_MAX_LEN);
1832 if (preboot_ver && preboot_ver != prop->preboot_ver) {
1833 strscpy(btl_ver, prop->preboot_ver,
1834 min((int) (preboot_ver - prop->preboot_ver),
1835 31));
1836 dev_info(hdev->dev, "%s\n", btl_ver);
1837 }
1838
1839 preboot_ver = extract_fw_ver_from_str(prop->preboot_ver);
1840 if (preboot_ver) {
1841 dev_info(hdev->dev, "preboot version %s\n",
1842 preboot_ver);
1843 kfree(preboot_ver);
1844 }
1845
1846 break;
1847 default:
1848 dev_warn(hdev->dev, "Undefined FW component: %d\n", fwc);
1849 return;
1850 }
1851 }
1852
1853 /**
1854 * hl_fw_dynamic_copy_image - copy image to memory allocated by the FW
1855 *
1856 * @hdev: pointer to the habanalabs device structure
1857 * @fw: fw descriptor
1858 * @fw_loader: managing structure for loading device's FW
1859 */
hl_fw_dynamic_copy_image(struct hl_device * hdev,const struct firmware * fw,struct fw_load_mgr * fw_loader)1860 static int hl_fw_dynamic_copy_image(struct hl_device *hdev,
1861 const struct firmware *fw,
1862 struct fw_load_mgr *fw_loader)
1863 {
1864 struct lkd_fw_comms_desc *fw_desc;
1865 struct pci_mem_region *region;
1866 void __iomem *dest;
1867 u64 addr;
1868 int rc;
1869
1870 fw_desc = &fw_loader->dynamic_loader.comm_desc;
1871 addr = le64_to_cpu(fw_desc->img_addr);
1872
1873 /* find memory region to which to copy the image */
1874 region = fw_loader->dynamic_loader.image_region;
1875
1876 dest = hdev->pcie_bar[region->bar_id] + region->offset_in_bar +
1877 (addr - region->region_base);
1878
1879 rc = hl_fw_copy_fw_to_device(hdev, fw, dest,
1880 fw_loader->boot_fit_img.src_off,
1881 fw_loader->boot_fit_img.copy_size);
1882
1883 return rc;
1884 }
1885
1886 /**
1887 * hl_fw_dynamic_copy_msg - copy msg to memory allocated by the FW
1888 *
1889 * @hdev: pointer to the habanalabs device structure
1890 * @msg: message
1891 * @fw_loader: managing structure for loading device's FW
1892 */
hl_fw_dynamic_copy_msg(struct hl_device * hdev,struct lkd_msg_comms * msg,struct fw_load_mgr * fw_loader)1893 static int hl_fw_dynamic_copy_msg(struct hl_device *hdev,
1894 struct lkd_msg_comms *msg, struct fw_load_mgr *fw_loader)
1895 {
1896 struct lkd_fw_comms_desc *fw_desc;
1897 struct pci_mem_region *region;
1898 void __iomem *dest;
1899 u64 addr;
1900 int rc;
1901
1902 fw_desc = &fw_loader->dynamic_loader.comm_desc;
1903 addr = le64_to_cpu(fw_desc->img_addr);
1904
1905 /* find memory region to which to copy the image */
1906 region = fw_loader->dynamic_loader.image_region;
1907
1908 dest = hdev->pcie_bar[region->bar_id] + region->offset_in_bar +
1909 (addr - region->region_base);
1910
1911 rc = hl_fw_copy_msg_to_device(hdev, msg, dest, 0, 0);
1912
1913 return rc;
1914 }
1915
1916 /**
1917 * hl_fw_boot_fit_update_state - update internal data structures after boot-fit
1918 * is loaded
1919 *
1920 * @hdev: pointer to the habanalabs device structure
1921 * @cpu_boot_dev_sts0_reg: register holding CPU boot dev status 0
1922 * @cpu_boot_dev_sts1_reg: register holding CPU boot dev status 1
1923 *
1924 * @return 0 on success, otherwise non-zero error code
1925 */
hl_fw_boot_fit_update_state(struct hl_device * hdev,u32 cpu_boot_dev_sts0_reg,u32 cpu_boot_dev_sts1_reg)1926 static void hl_fw_boot_fit_update_state(struct hl_device *hdev,
1927 u32 cpu_boot_dev_sts0_reg,
1928 u32 cpu_boot_dev_sts1_reg)
1929 {
1930 struct asic_fixed_properties *prop = &hdev->asic_prop;
1931
1932 /* Clear reset status since we need to read it again from boot CPU */
1933 prop->hard_reset_done_by_fw = false;
1934
1935 /* Read boot_cpu status bits */
1936 if (prop->fw_preboot_cpu_boot_dev_sts0 & CPU_BOOT_DEV_STS0_ENABLED) {
1937 prop->fw_bootfit_cpu_boot_dev_sts0 =
1938 RREG32(cpu_boot_dev_sts0_reg);
1939
1940 if (prop->fw_bootfit_cpu_boot_dev_sts0 &
1941 CPU_BOOT_DEV_STS0_FW_HARD_RST_EN)
1942 prop->hard_reset_done_by_fw = true;
1943
1944 dev_dbg(hdev->dev, "Firmware boot CPU status0 %#x\n",
1945 prop->fw_bootfit_cpu_boot_dev_sts0);
1946 }
1947
1948 if (prop->fw_cpu_boot_dev_sts1_valid) {
1949 prop->fw_bootfit_cpu_boot_dev_sts1 =
1950 RREG32(cpu_boot_dev_sts1_reg);
1951
1952 dev_dbg(hdev->dev, "Firmware boot CPU status1 %#x\n",
1953 prop->fw_bootfit_cpu_boot_dev_sts1);
1954 }
1955
1956 dev_dbg(hdev->dev, "Firmware boot CPU hard-reset is %s\n",
1957 prop->hard_reset_done_by_fw ? "enabled" : "disabled");
1958 }
1959
hl_fw_dynamic_update_linux_interrupt_if(struct hl_device * hdev)1960 static void hl_fw_dynamic_update_linux_interrupt_if(struct hl_device *hdev)
1961 {
1962 struct cpu_dyn_regs *dyn_regs =
1963 &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;
1964
1965 /* Check whether all 3 interrupt interfaces are set, if not use a
1966 * single interface
1967 */
1968 if (!hdev->asic_prop.gic_interrupts_enable &&
1969 !(hdev->asic_prop.fw_app_cpu_boot_dev_sts0 &
1970 CPU_BOOT_DEV_STS0_MULTI_IRQ_POLL_EN)) {
1971 dyn_regs->gic_host_halt_irq = dyn_regs->gic_host_pi_upd_irq;
1972 dyn_regs->gic_host_ints_irq = dyn_regs->gic_host_pi_upd_irq;
1973
1974 dev_warn(hdev->dev,
1975 "Using a single interrupt interface towards cpucp");
1976 }
1977 }
1978 /**
1979 * hl_fw_dynamic_load_image - load FW image using dynamic protocol
1980 *
1981 * @hdev: pointer to the habanalabs device structure
1982 * @fw_loader: managing structure for loading device's FW
1983 * @load_fwc: the FW component to be loaded
1984 * @img_ld_timeout: image load timeout
1985 *
1986 * @return 0 on success, otherwise non-zero error code
1987 */
hl_fw_dynamic_load_image(struct hl_device * hdev,struct fw_load_mgr * fw_loader,enum hl_fw_component load_fwc,u32 img_ld_timeout)1988 static int hl_fw_dynamic_load_image(struct hl_device *hdev,
1989 struct fw_load_mgr *fw_loader,
1990 enum hl_fw_component load_fwc,
1991 u32 img_ld_timeout)
1992 {
1993 enum hl_fw_component cur_fwc;
1994 const struct firmware *fw;
1995 char *fw_name;
1996 int rc = 0;
1997
1998 /*
1999 * when loading image we have one of 2 scenarios:
2000 * 1. current FW component is preboot and we want to load boot-fit
2001 * 2. current FW component is boot-fit and we want to load linux
2002 */
2003 if (load_fwc == FW_COMP_BOOT_FIT) {
2004 cur_fwc = FW_COMP_PREBOOT;
2005 fw_name = fw_loader->boot_fit_img.image_name;
2006 } else {
2007 cur_fwc = FW_COMP_BOOT_FIT;
2008 fw_name = fw_loader->linux_img.image_name;
2009 }
2010
2011 /* request FW in order to communicate to FW the size to be allocated */
2012 rc = hl_request_fw(hdev, &fw, fw_name);
2013 if (rc)
2014 return rc;
2015
2016 /* store the image size for future validation */
2017 fw_loader->dynamic_loader.fw_image_size = fw->size;
2018
2019 rc = hl_fw_dynamic_request_descriptor(hdev, fw_loader, fw->size);
2020 if (rc)
2021 goto release_fw;
2022
2023 /* read preboot version */
2024 hl_fw_dynamic_read_device_fw_version(hdev, cur_fwc,
2025 fw_loader->dynamic_loader.comm_desc.cur_fw_ver);
2026
2027
2028 /* update state according to boot stage */
2029 if (cur_fwc == FW_COMP_BOOT_FIT) {
2030 struct cpu_dyn_regs *dyn_regs;
2031
2032 dyn_regs = &fw_loader->dynamic_loader.comm_desc.cpu_dyn_regs;
2033 hl_fw_boot_fit_update_state(hdev,
2034 le32_to_cpu(dyn_regs->cpu_boot_dev_sts0),
2035 le32_to_cpu(dyn_regs->cpu_boot_dev_sts1));
2036 }
2037
2038 /* copy boot fit to space allocated by FW */
2039 rc = hl_fw_dynamic_copy_image(hdev, fw, fw_loader);
2040 if (rc)
2041 goto release_fw;
2042
2043 rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_DATA_RDY,
2044 0, true,
2045 fw_loader->cpu_timeout);
2046 if (rc)
2047 goto release_fw;
2048
2049 rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_EXEC,
2050 0, false,
2051 img_ld_timeout);
2052
2053 release_fw:
2054 hl_release_firmware(fw);
2055 return rc;
2056 }
2057
hl_fw_dynamic_wait_for_boot_fit_active(struct hl_device * hdev,struct fw_load_mgr * fw_loader)2058 static int hl_fw_dynamic_wait_for_boot_fit_active(struct hl_device *hdev,
2059 struct fw_load_mgr *fw_loader)
2060 {
2061 struct dynamic_fw_load_mgr *dyn_loader;
2062 u32 status;
2063 int rc;
2064
2065 dyn_loader = &fw_loader->dynamic_loader;
2066
2067 /* Make sure CPU boot-loader is running */
2068 rc = hl_poll_timeout(
2069 hdev,
2070 le32_to_cpu(dyn_loader->comm_desc.cpu_dyn_regs.cpu_boot_status),
2071 status,
2072 (status == CPU_BOOT_STATUS_NIC_FW_RDY) ||
2073 (status == CPU_BOOT_STATUS_READY_TO_BOOT),
2074 FW_CPU_STATUS_POLL_INTERVAL_USEC,
2075 dyn_loader->wait_for_bl_timeout);
2076 if (rc) {
2077 dev_err(hdev->dev, "failed to wait for boot\n");
2078 return rc;
2079 }
2080
2081 dev_dbg(hdev->dev, "uboot status = %d\n", status);
2082 return 0;
2083 }
2084
hl_fw_dynamic_wait_for_linux_active(struct hl_device * hdev,struct fw_load_mgr * fw_loader)2085 static int hl_fw_dynamic_wait_for_linux_active(struct hl_device *hdev,
2086 struct fw_load_mgr *fw_loader)
2087 {
2088 struct dynamic_fw_load_mgr *dyn_loader;
2089 u32 status;
2090 int rc;
2091
2092 dyn_loader = &fw_loader->dynamic_loader;
2093
2094 /* Make sure CPU boot-loader is running */
2095
2096 rc = hl_poll_timeout(
2097 hdev,
2098 le32_to_cpu(dyn_loader->comm_desc.cpu_dyn_regs.cpu_boot_status),
2099 status,
2100 (status == CPU_BOOT_STATUS_SRAM_AVAIL),
2101 FW_CPU_STATUS_POLL_INTERVAL_USEC,
2102 fw_loader->cpu_timeout);
2103 if (rc) {
2104 dev_err(hdev->dev, "failed to wait for Linux\n");
2105 return rc;
2106 }
2107
2108 dev_dbg(hdev->dev, "Boot status = %d\n", status);
2109 return 0;
2110 }
2111
2112 /**
2113 * hl_fw_linux_update_state - update internal data structures after Linux
2114 * is loaded.
2115 * Note: Linux initialization is comprised mainly
2116 * of two stages - loading kernel (SRAM_AVAIL)
2117 * & loading ARMCP.
2118 * Therefore reading boot device status in any of
2119 * these stages might result in different values.
2120 *
2121 * @hdev: pointer to the habanalabs device structure
2122 * @cpu_boot_dev_sts0_reg: register holding CPU boot dev status 0
2123 * @cpu_boot_dev_sts1_reg: register holding CPU boot dev status 1
2124 *
2125 * @return 0 on success, otherwise non-zero error code
2126 */
hl_fw_linux_update_state(struct hl_device * hdev,u32 cpu_boot_dev_sts0_reg,u32 cpu_boot_dev_sts1_reg)2127 static void hl_fw_linux_update_state(struct hl_device *hdev,
2128 u32 cpu_boot_dev_sts0_reg,
2129 u32 cpu_boot_dev_sts1_reg)
2130 {
2131 struct asic_fixed_properties *prop = &hdev->asic_prop;
2132
2133 hdev->fw_loader.linux_loaded = true;
2134
2135 /* Clear reset status since we need to read again from app */
2136 prop->hard_reset_done_by_fw = false;
2137
2138 /* Read FW application security bits */
2139 if (prop->fw_cpu_boot_dev_sts0_valid) {
2140 prop->fw_app_cpu_boot_dev_sts0 = RREG32(cpu_boot_dev_sts0_reg);
2141
2142 if (prop->fw_app_cpu_boot_dev_sts0 &
2143 CPU_BOOT_DEV_STS0_FW_HARD_RST_EN)
2144 prop->hard_reset_done_by_fw = true;
2145
2146 if (prop->fw_app_cpu_boot_dev_sts0 &
2147 CPU_BOOT_DEV_STS0_GIC_PRIVILEGED_EN)
2148 prop->gic_interrupts_enable = false;
2149
2150 dev_dbg(hdev->dev,
2151 "Firmware application CPU status0 %#x\n",
2152 prop->fw_app_cpu_boot_dev_sts0);
2153
2154 dev_dbg(hdev->dev, "GIC controller is %s\n",
2155 prop->gic_interrupts_enable ?
2156 "enabled" : "disabled");
2157 }
2158
2159 if (prop->fw_cpu_boot_dev_sts1_valid) {
2160 prop->fw_app_cpu_boot_dev_sts1 = RREG32(cpu_boot_dev_sts1_reg);
2161
2162 dev_dbg(hdev->dev,
2163 "Firmware application CPU status1 %#x\n",
2164 prop->fw_app_cpu_boot_dev_sts1);
2165 }
2166
2167 dev_dbg(hdev->dev, "Firmware application CPU hard-reset is %s\n",
2168 prop->hard_reset_done_by_fw ? "enabled" : "disabled");
2169
2170 dev_info(hdev->dev, "Successfully loaded firmware to device\n");
2171 }
2172
2173 /**
2174 * hl_fw_dynamic_report_reset_cause - send a COMMS message with the cause
2175 * of the newly triggered hard reset
2176 *
2177 * @hdev: pointer to the habanalabs device structure
2178 * @fw_loader: managing structure for loading device's FW
2179 * @reset_cause: enumerated cause for the recent hard reset
2180 *
2181 * @return 0 on success, otherwise non-zero error code
2182 */
hl_fw_dynamic_report_reset_cause(struct hl_device * hdev,struct fw_load_mgr * fw_loader,enum comms_reset_cause reset_cause)2183 static int hl_fw_dynamic_report_reset_cause(struct hl_device *hdev,
2184 struct fw_load_mgr *fw_loader,
2185 enum comms_reset_cause reset_cause)
2186 {
2187 struct lkd_msg_comms msg;
2188 int rc;
2189
2190 memset(&msg, 0, sizeof(msg));
2191
2192 /* create message to be sent */
2193 msg.header.type = HL_COMMS_RESET_CAUSE_TYPE;
2194 msg.header.size = cpu_to_le16(sizeof(struct comms_msg_header));
2195 msg.header.magic = cpu_to_le32(HL_COMMS_MSG_MAGIC);
2196
2197 msg.reset_cause = reset_cause;
2198
2199 rc = hl_fw_dynamic_request_descriptor(hdev, fw_loader,
2200 sizeof(struct lkd_msg_comms));
2201 if (rc)
2202 return rc;
2203
2204 /* copy message to space allocated by FW */
2205 rc = hl_fw_dynamic_copy_msg(hdev, &msg, fw_loader);
2206 if (rc)
2207 return rc;
2208
2209 rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_DATA_RDY,
2210 0, true,
2211 fw_loader->cpu_timeout);
2212 if (rc)
2213 return rc;
2214
2215 rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_EXEC,
2216 0, true,
2217 fw_loader->cpu_timeout);
2218 if (rc)
2219 return rc;
2220
2221 return 0;
2222 }
2223
2224 /**
2225 * hl_fw_dynamic_init_cpu - initialize the device CPU using dynamic protocol
2226 *
2227 * @hdev: pointer to the habanalabs device structure
2228 * @fw_loader: managing structure for loading device's FW
2229 *
2230 * @return 0 on success, otherwise non-zero error code
2231 *
2232 * brief: the dynamic protocol is master (LKD) slave (FW CPU) protocol.
2233 * the communication is done using registers:
2234 * - LKD command register
2235 * - FW status register
2236 * the protocol is race free. this goal is achieved by splitting the requests
2237 * and response to known synchronization points between the LKD and the FW.
2238 * each response to LKD request is known and bound to a predefined timeout.
2239 * in case of timeout expiration without the desired status from FW- the
2240 * protocol (and hence the boot) will fail.
2241 */
hl_fw_dynamic_init_cpu(struct hl_device * hdev,struct fw_load_mgr * fw_loader)2242 static int hl_fw_dynamic_init_cpu(struct hl_device *hdev,
2243 struct fw_load_mgr *fw_loader)
2244 {
2245 struct cpu_dyn_regs *dyn_regs;
2246 int rc;
2247
2248 dev_info(hdev->dev,
2249 "Loading firmware to device, may take some time...\n");
2250
2251 /* initialize FW descriptor as invalid */
2252 fw_loader->dynamic_loader.fw_desc_valid = false;
2253
2254 /*
2255 * In this stage, "cpu_dyn_regs" contains only LKD's hard coded values!
2256 * It will be updated from FW after hl_fw_dynamic_request_descriptor().
2257 */
2258 dyn_regs = &fw_loader->dynamic_loader.comm_desc.cpu_dyn_regs;
2259
2260 rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_RST_STATE,
2261 0, true,
2262 fw_loader->cpu_timeout);
2263 if (rc)
2264 goto protocol_err;
2265
2266 if (hdev->curr_reset_cause) {
2267 rc = hl_fw_dynamic_report_reset_cause(hdev, fw_loader,
2268 hdev->curr_reset_cause);
2269 if (rc)
2270 goto protocol_err;
2271
2272 /* Clear current reset cause */
2273 hdev->curr_reset_cause = HL_RESET_CAUSE_UNKNOWN;
2274 }
2275
2276 if (!(hdev->fw_components & FW_TYPE_BOOT_CPU)) {
2277 rc = hl_fw_dynamic_request_descriptor(hdev, fw_loader, 0);
2278 if (rc)
2279 goto protocol_err;
2280
2281 /* read preboot version */
2282 hl_fw_dynamic_read_device_fw_version(hdev, FW_COMP_PREBOOT,
2283 fw_loader->dynamic_loader.comm_desc.cur_fw_ver);
2284 return 0;
2285 }
2286
2287 /* load boot fit to FW */
2288 rc = hl_fw_dynamic_load_image(hdev, fw_loader, FW_COMP_BOOT_FIT,
2289 fw_loader->boot_fit_timeout);
2290 if (rc) {
2291 dev_err(hdev->dev, "failed to load boot fit\n");
2292 goto protocol_err;
2293 }
2294
2295 rc = hl_fw_dynamic_wait_for_boot_fit_active(hdev, fw_loader);
2296 if (rc)
2297 goto protocol_err;
2298
2299 /* Enable DRAM scrambling before Linux boot and after successful
2300 * UBoot
2301 */
2302 hdev->asic_funcs->init_cpu_scrambler_dram(hdev);
2303
2304 if (!(hdev->fw_components & FW_TYPE_LINUX)) {
2305 dev_info(hdev->dev, "Skip loading Linux F/W\n");
2306 return 0;
2307 }
2308
2309 if (fw_loader->skip_bmc) {
2310 rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader,
2311 COMMS_SKIP_BMC, 0,
2312 true,
2313 fw_loader->cpu_timeout);
2314 if (rc) {
2315 dev_err(hdev->dev, "failed to load boot fit\n");
2316 goto protocol_err;
2317 }
2318 }
2319
2320 /* load Linux image to FW */
2321 rc = hl_fw_dynamic_load_image(hdev, fw_loader, FW_COMP_LINUX,
2322 fw_loader->cpu_timeout);
2323 if (rc) {
2324 dev_err(hdev->dev, "failed to load Linux\n");
2325 goto protocol_err;
2326 }
2327
2328 rc = hl_fw_dynamic_wait_for_linux_active(hdev, fw_loader);
2329 if (rc)
2330 goto protocol_err;
2331
2332 hl_fw_linux_update_state(hdev, le32_to_cpu(dyn_regs->cpu_boot_dev_sts0),
2333 le32_to_cpu(dyn_regs->cpu_boot_dev_sts1));
2334
2335 hl_fw_dynamic_update_linux_interrupt_if(hdev);
2336
2337 return 0;
2338
2339 protocol_err:
2340 if (fw_loader->dynamic_loader.fw_desc_valid)
2341 fw_read_errors(hdev, le32_to_cpu(dyn_regs->cpu_boot_err0),
2342 le32_to_cpu(dyn_regs->cpu_boot_err1),
2343 le32_to_cpu(dyn_regs->cpu_boot_dev_sts0),
2344 le32_to_cpu(dyn_regs->cpu_boot_dev_sts1));
2345 return rc;
2346 }
2347
2348 /**
2349 * hl_fw_static_init_cpu - initialize the device CPU using static protocol
2350 *
2351 * @hdev: pointer to the habanalabs device structure
2352 * @fw_loader: managing structure for loading device's FW
2353 *
2354 * @return 0 on success, otherwise non-zero error code
2355 */
hl_fw_static_init_cpu(struct hl_device * hdev,struct fw_load_mgr * fw_loader)2356 static int hl_fw_static_init_cpu(struct hl_device *hdev,
2357 struct fw_load_mgr *fw_loader)
2358 {
2359 u32 cpu_msg_status_reg, cpu_timeout, msg_to_cpu_reg, status;
2360 u32 cpu_boot_dev_status0_reg, cpu_boot_dev_status1_reg;
2361 struct static_fw_load_mgr *static_loader;
2362 u32 cpu_boot_status_reg;
2363 int rc;
2364
2365 if (!(hdev->fw_components & FW_TYPE_BOOT_CPU))
2366 return 0;
2367
2368 /* init common loader parameters */
2369 cpu_timeout = fw_loader->cpu_timeout;
2370
2371 /* init static loader parameters */
2372 static_loader = &fw_loader->static_loader;
2373 cpu_msg_status_reg = static_loader->cpu_cmd_status_to_host_reg;
2374 msg_to_cpu_reg = static_loader->kmd_msg_to_cpu_reg;
2375 cpu_boot_dev_status0_reg = static_loader->cpu_boot_dev_status0_reg;
2376 cpu_boot_dev_status1_reg = static_loader->cpu_boot_dev_status1_reg;
2377 cpu_boot_status_reg = static_loader->cpu_boot_status_reg;
2378
2379 dev_info(hdev->dev, "Going to wait for device boot (up to %lds)\n",
2380 cpu_timeout / USEC_PER_SEC);
2381
2382 /* Wait for boot FIT request */
2383 rc = hl_poll_timeout(
2384 hdev,
2385 cpu_boot_status_reg,
2386 status,
2387 status == CPU_BOOT_STATUS_WAITING_FOR_BOOT_FIT,
2388 FW_CPU_STATUS_POLL_INTERVAL_USEC,
2389 fw_loader->boot_fit_timeout);
2390
2391 if (rc) {
2392 dev_dbg(hdev->dev,
2393 "No boot fit request received, resuming boot\n");
2394 } else {
2395 rc = hdev->asic_funcs->load_boot_fit_to_device(hdev);
2396 if (rc)
2397 goto out;
2398
2399 /* Clear device CPU message status */
2400 WREG32(cpu_msg_status_reg, CPU_MSG_CLR);
2401
2402 /* Signal device CPU that boot loader is ready */
2403 WREG32(msg_to_cpu_reg, KMD_MSG_FIT_RDY);
2404
2405 /* Poll for CPU device ack */
2406 rc = hl_poll_timeout(
2407 hdev,
2408 cpu_msg_status_reg,
2409 status,
2410 status == CPU_MSG_OK,
2411 FW_CPU_STATUS_POLL_INTERVAL_USEC,
2412 fw_loader->boot_fit_timeout);
2413
2414 if (rc) {
2415 dev_err(hdev->dev,
2416 "Timeout waiting for boot fit load ack\n");
2417 goto out;
2418 }
2419
2420 /* Clear message */
2421 WREG32(msg_to_cpu_reg, KMD_MSG_NA);
2422 }
2423
2424 /* Make sure CPU boot-loader is running */
2425 rc = hl_poll_timeout(
2426 hdev,
2427 cpu_boot_status_reg,
2428 status,
2429 (status == CPU_BOOT_STATUS_DRAM_RDY) ||
2430 (status == CPU_BOOT_STATUS_NIC_FW_RDY) ||
2431 (status == CPU_BOOT_STATUS_READY_TO_BOOT) ||
2432 (status == CPU_BOOT_STATUS_SRAM_AVAIL),
2433 FW_CPU_STATUS_POLL_INTERVAL_USEC,
2434 cpu_timeout);
2435
2436 dev_dbg(hdev->dev, "uboot status = %d\n", status);
2437
2438 /* Read U-Boot version now in case we will later fail */
2439 hl_fw_static_read_device_fw_version(hdev, FW_COMP_BOOT_FIT);
2440
2441 /* update state according to boot stage */
2442 hl_fw_boot_fit_update_state(hdev, cpu_boot_dev_status0_reg,
2443 cpu_boot_dev_status1_reg);
2444
2445 if (rc) {
2446 detect_cpu_boot_status(hdev, status);
2447 rc = -EIO;
2448 goto out;
2449 }
2450
2451 /* Enable DRAM scrambling before Linux boot and after successful
2452 * UBoot
2453 */
2454 hdev->asic_funcs->init_cpu_scrambler_dram(hdev);
2455
2456 if (!(hdev->fw_components & FW_TYPE_LINUX)) {
2457 dev_info(hdev->dev, "Skip loading Linux F/W\n");
2458 rc = 0;
2459 goto out;
2460 }
2461
2462 if (status == CPU_BOOT_STATUS_SRAM_AVAIL) {
2463 rc = 0;
2464 goto out;
2465 }
2466
2467 dev_info(hdev->dev,
2468 "Loading firmware to device, may take some time...\n");
2469
2470 rc = hdev->asic_funcs->load_firmware_to_device(hdev);
2471 if (rc)
2472 goto out;
2473
2474 if (fw_loader->skip_bmc) {
2475 WREG32(msg_to_cpu_reg, KMD_MSG_SKIP_BMC);
2476
2477 rc = hl_poll_timeout(
2478 hdev,
2479 cpu_boot_status_reg,
2480 status,
2481 (status == CPU_BOOT_STATUS_BMC_WAITING_SKIPPED),
2482 FW_CPU_STATUS_POLL_INTERVAL_USEC,
2483 cpu_timeout);
2484
2485 if (rc) {
2486 dev_err(hdev->dev,
2487 "Failed to get ACK on skipping BMC, %d\n",
2488 status);
2489 WREG32(msg_to_cpu_reg, KMD_MSG_NA);
2490 rc = -EIO;
2491 goto out;
2492 }
2493 }
2494
2495 WREG32(msg_to_cpu_reg, KMD_MSG_FIT_RDY);
2496
2497 rc = hl_poll_timeout(
2498 hdev,
2499 cpu_boot_status_reg,
2500 status,
2501 (status == CPU_BOOT_STATUS_SRAM_AVAIL),
2502 FW_CPU_STATUS_POLL_INTERVAL_USEC,
2503 cpu_timeout);
2504
2505 /* Clear message */
2506 WREG32(msg_to_cpu_reg, KMD_MSG_NA);
2507
2508 if (rc) {
2509 if (status == CPU_BOOT_STATUS_FIT_CORRUPTED)
2510 dev_err(hdev->dev,
2511 "Device reports FIT image is corrupted\n");
2512 else
2513 dev_err(hdev->dev,
2514 "Failed to load firmware to device, %d\n",
2515 status);
2516
2517 rc = -EIO;
2518 goto out;
2519 }
2520
2521 rc = fw_read_errors(hdev, fw_loader->static_loader.boot_err0_reg,
2522 fw_loader->static_loader.boot_err1_reg,
2523 cpu_boot_dev_status0_reg,
2524 cpu_boot_dev_status1_reg);
2525 if (rc)
2526 return rc;
2527
2528 hl_fw_linux_update_state(hdev, cpu_boot_dev_status0_reg,
2529 cpu_boot_dev_status1_reg);
2530
2531 return 0;
2532
2533 out:
2534 fw_read_errors(hdev, fw_loader->static_loader.boot_err0_reg,
2535 fw_loader->static_loader.boot_err1_reg,
2536 cpu_boot_dev_status0_reg,
2537 cpu_boot_dev_status1_reg);
2538
2539 return rc;
2540 }
2541
2542 /**
2543 * hl_fw_init_cpu - initialize the device CPU
2544 *
2545 * @hdev: pointer to the habanalabs device structure
2546 *
2547 * @return 0 on success, otherwise non-zero error code
2548 *
2549 * perform necessary initializations for device's CPU. takes into account if
2550 * init protocol is static or dynamic.
2551 */
hl_fw_init_cpu(struct hl_device * hdev)2552 int hl_fw_init_cpu(struct hl_device *hdev)
2553 {
2554 struct asic_fixed_properties *prop = &hdev->asic_prop;
2555 struct fw_load_mgr *fw_loader = &hdev->fw_loader;
2556
2557 return prop->dynamic_fw_load ?
2558 hl_fw_dynamic_init_cpu(hdev, fw_loader) :
2559 hl_fw_static_init_cpu(hdev, fw_loader);
2560 }
2561