1 // SPDX-License-Identifier: GPL-2.0
2
3 /*
4 * Copyright 2016-2022 HabanaLabs, Ltd.
5 * All Rights Reserved.
6 */
7
8 #include "habanalabs.h"
9
10 #include <linux/slab.h>
11
12 /**
13 * struct hl_eqe_work - This structure is used to schedule work of EQ
14 * entry and cpucp_reset event
15 *
16 * @eq_work: workqueue object to run when EQ entry is received
17 * @hdev: pointer to device structure
18 * @eq_entry: copy of the EQ entry
19 */
20 struct hl_eqe_work {
21 struct work_struct eq_work;
22 struct hl_device *hdev;
23 struct hl_eq_entry eq_entry;
24 };
25
26 /**
27 * hl_cq_inc_ptr - increment ci or pi of cq
28 *
29 * @ptr: the current ci or pi value of the completion queue
30 *
31 * Increment ptr by 1. If it reaches the number of completion queue
32 * entries, set it to 0
33 */
hl_cq_inc_ptr(u32 ptr)34 inline u32 hl_cq_inc_ptr(u32 ptr)
35 {
36 ptr++;
37 if (unlikely(ptr == HL_CQ_LENGTH))
38 ptr = 0;
39 return ptr;
40 }
41
42 /**
43 * hl_eq_inc_ptr - increment ci of eq
44 *
45 * @ptr: the current ci value of the event queue
46 *
47 * Increment ptr by 1. If it reaches the number of event queue
48 * entries, set it to 0
49 */
hl_eq_inc_ptr(u32 ptr)50 static inline u32 hl_eq_inc_ptr(u32 ptr)
51 {
52 ptr++;
53 if (unlikely(ptr == HL_EQ_LENGTH))
54 ptr = 0;
55 return ptr;
56 }
57
irq_handle_eqe(struct work_struct * work)58 static void irq_handle_eqe(struct work_struct *work)
59 {
60 struct hl_eqe_work *eqe_work = container_of(work, struct hl_eqe_work,
61 eq_work);
62 struct hl_device *hdev = eqe_work->hdev;
63
64 hdev->asic_funcs->handle_eqe(hdev, &eqe_work->eq_entry);
65
66 kfree(eqe_work);
67 }
68
69 /**
70 * job_finish - queue job finish work
71 *
72 * @hdev: pointer to device structure
73 * @cs_seq: command submission sequence
74 * @cq: completion queue
75 * @timestamp: interrupt timestamp
76 *
77 */
job_finish(struct hl_device * hdev,u32 cs_seq,struct hl_cq * cq,ktime_t timestamp)78 static void job_finish(struct hl_device *hdev, u32 cs_seq, struct hl_cq *cq, ktime_t timestamp)
79 {
80 struct hl_hw_queue *queue;
81 struct hl_cs_job *job;
82
83 queue = &hdev->kernel_queues[cq->hw_queue_id];
84 job = queue->shadow_queue[hl_pi_2_offset(cs_seq)];
85 job->timestamp = timestamp;
86 queue_work(hdev->cq_wq[cq->cq_idx], &job->finish_work);
87
88 atomic_inc(&queue->ci);
89 }
90
91 /**
92 * cs_finish - queue all cs jobs finish work
93 *
94 * @hdev: pointer to device structure
95 * @cs_seq: command submission sequence
96 * @timestamp: interrupt timestamp
97 *
98 */
cs_finish(struct hl_device * hdev,u16 cs_seq,ktime_t timestamp)99 static void cs_finish(struct hl_device *hdev, u16 cs_seq, ktime_t timestamp)
100 {
101 struct asic_fixed_properties *prop = &hdev->asic_prop;
102 struct hl_hw_queue *queue;
103 struct hl_cs *cs;
104 struct hl_cs_job *job;
105
106 cs = hdev->shadow_cs_queue[cs_seq & (prop->max_pending_cs - 1)];
107 if (!cs) {
108 dev_warn(hdev->dev,
109 "No pointer to CS in shadow array at index %d\n",
110 cs_seq);
111 return;
112 }
113
114 list_for_each_entry(job, &cs->job_list, cs_node) {
115 queue = &hdev->kernel_queues[job->hw_queue_id];
116 atomic_inc(&queue->ci);
117 }
118
119 cs->completion_timestamp = timestamp;
120 queue_work(hdev->cs_cmplt_wq, &cs->finish_work);
121 }
122
123 /**
124 * hl_irq_handler_cq - irq handler for completion queue
125 *
126 * @irq: irq number
127 * @arg: pointer to completion queue structure
128 *
129 */
hl_irq_handler_cq(int irq,void * arg)130 irqreturn_t hl_irq_handler_cq(int irq, void *arg)
131 {
132 struct hl_cq *cq = arg;
133 struct hl_device *hdev = cq->hdev;
134 bool shadow_index_valid, entry_ready;
135 u16 shadow_index;
136 struct hl_cq_entry *cq_entry, *cq_base;
137 ktime_t timestamp = ktime_get();
138
139 if (hdev->disabled) {
140 dev_dbg(hdev->dev,
141 "Device disabled but received IRQ %d for CQ %d\n",
142 irq, cq->hw_queue_id);
143 return IRQ_HANDLED;
144 }
145
146 cq_base = cq->kernel_address;
147
148 while (1) {
149 cq_entry = (struct hl_cq_entry *) &cq_base[cq->ci];
150
151 entry_ready = !!FIELD_GET(CQ_ENTRY_READY_MASK,
152 le32_to_cpu(cq_entry->data));
153 if (!entry_ready)
154 break;
155
156 /* Make sure we read CQ entry contents after we've
157 * checked the ownership bit.
158 */
159 dma_rmb();
160
161 shadow_index_valid =
162 !!FIELD_GET(CQ_ENTRY_SHADOW_INDEX_VALID_MASK,
163 le32_to_cpu(cq_entry->data));
164
165 shadow_index = FIELD_GET(CQ_ENTRY_SHADOW_INDEX_MASK,
166 le32_to_cpu(cq_entry->data));
167
168 /*
169 * CQ interrupt handler has 2 modes of operation:
170 * 1. Interrupt per CS completion: (Single CQ for all queues)
171 * CQ entry represents a completed CS
172 *
173 * 2. Interrupt per CS job completion in queue: (CQ per queue)
174 * CQ entry represents a completed job in a certain queue
175 */
176 if (shadow_index_valid && !hdev->disabled) {
177 if (hdev->asic_prop.completion_mode ==
178 HL_COMPLETION_MODE_CS)
179 cs_finish(hdev, shadow_index, timestamp);
180 else
181 job_finish(hdev, shadow_index, cq, timestamp);
182 }
183
184 /* Clear CQ entry ready bit */
185 cq_entry->data = cpu_to_le32(le32_to_cpu(cq_entry->data) &
186 ~CQ_ENTRY_READY_MASK);
187
188 cq->ci = hl_cq_inc_ptr(cq->ci);
189
190 /* Increment free slots */
191 atomic_inc(&cq->free_slots_cnt);
192 }
193
194 return IRQ_HANDLED;
195 }
196
197 /*
198 * hl_ts_free_objects - handler of the free objects workqueue.
199 * This function should put refcount to objects that the registration node
200 * took refcount to them.
201 * @work: workqueue object pointer
202 */
hl_ts_free_objects(struct work_struct * work)203 static void hl_ts_free_objects(struct work_struct *work)
204 {
205 struct timestamp_reg_work_obj *job =
206 container_of(work, struct timestamp_reg_work_obj, free_obj);
207 struct timestamp_reg_free_node *free_obj, *temp_free_obj;
208 struct list_head *free_list_head = job->free_obj_head;
209 struct hl_device *hdev = job->hdev;
210
211 list_for_each_entry_safe(free_obj, temp_free_obj, free_list_head, free_objects_node) {
212 dev_dbg(hdev->dev, "About to put refcount to buf (%p) cq_cb(%p)\n",
213 free_obj->buf,
214 free_obj->cq_cb);
215
216 hl_mmap_mem_buf_put(free_obj->buf);
217 hl_cb_put(free_obj->cq_cb);
218 kfree(free_obj);
219 }
220
221 kfree(free_list_head);
222 kfree(job);
223 }
224
225 /*
226 * This function called with spin_lock of wait_list_lock taken
227 * This function will set timestamp and delete the registration node from the
228 * wait_list_lock.
229 * and since we're protected with spin_lock here, so we cannot just put the refcount
230 * for the objects here, since the release function may be called and it's also a long
231 * logic (which might sleep also) that cannot be handled in irq context.
232 * so here we'll be filling a list with nodes of "put" jobs and then will send this
233 * list to a dedicated workqueue to do the actual put.
234 */
handle_registration_node(struct hl_device * hdev,struct hl_user_pending_interrupt * pend,struct list_head ** free_list,ktime_t now)235 static int handle_registration_node(struct hl_device *hdev, struct hl_user_pending_interrupt *pend,
236 struct list_head **free_list, ktime_t now)
237 {
238 struct timestamp_reg_free_node *free_node;
239 u64 timestamp;
240
241 if (!(*free_list)) {
242 /* Alloc/Init the timestamp registration free objects list */
243 *free_list = kmalloc(sizeof(struct list_head), GFP_ATOMIC);
244 if (!(*free_list))
245 return -ENOMEM;
246
247 INIT_LIST_HEAD(*free_list);
248 }
249
250 free_node = kmalloc(sizeof(*free_node), GFP_ATOMIC);
251 if (!free_node)
252 return -ENOMEM;
253
254 timestamp = ktime_to_ns(now);
255
256 *pend->ts_reg_info.timestamp_kernel_addr = timestamp;
257
258 dev_dbg(hdev->dev, "Timestamp is set to ts cb address (%p), ts: 0x%llx\n",
259 pend->ts_reg_info.timestamp_kernel_addr,
260 *(u64 *)pend->ts_reg_info.timestamp_kernel_addr);
261
262 list_del(&pend->wait_list_node);
263
264 /* Mark kernel CB node as free */
265 pend->ts_reg_info.in_use = 0;
266
267 /* Putting the refcount for ts_buff and cq_cb objects will be handled
268 * in workqueue context, just add job to free_list.
269 */
270 free_node->buf = pend->ts_reg_info.buf;
271 free_node->cq_cb = pend->ts_reg_info.cq_cb;
272 list_add(&free_node->free_objects_node, *free_list);
273
274 /* Mark TS record as free */
275 pend->ts_reg_info.in_use = false;
276
277 return 0;
278 }
279
handle_user_interrupt(struct hl_device * hdev,struct hl_user_interrupt * intr)280 static void handle_user_interrupt(struct hl_device *hdev, struct hl_user_interrupt *intr)
281 {
282 struct hl_user_pending_interrupt *pend, *temp_pend;
283 struct list_head *ts_reg_free_list_head = NULL;
284 struct timestamp_reg_work_obj *job;
285 bool reg_node_handle_fail = false;
286 int rc;
287
288 /* For registration nodes:
289 * As part of handling the registration nodes, we should put refcount to
290 * some objects. the problem is that we cannot do that under spinlock
291 * or in irq handler context at all (since release functions are long and
292 * might sleep), so we will need to handle that part in workqueue context.
293 * To avoid handling kmalloc failure which compels us rolling back actions
294 * and move nodes hanged on the free list back to the interrupt wait list
295 * we always alloc the job of the WQ at the beginning.
296 */
297 job = kmalloc(sizeof(*job), GFP_ATOMIC);
298 if (!job)
299 return;
300
301 spin_lock(&intr->wait_list_lock);
302 list_for_each_entry_safe(pend, temp_pend, &intr->wait_list_head, wait_list_node) {
303 if ((pend->cq_kernel_addr && *(pend->cq_kernel_addr) >= pend->cq_target_value) ||
304 !pend->cq_kernel_addr) {
305 if (pend->ts_reg_info.buf) {
306 if (!reg_node_handle_fail) {
307 rc = handle_registration_node(hdev, pend,
308 &ts_reg_free_list_head, intr->timestamp);
309 if (rc)
310 reg_node_handle_fail = true;
311 }
312 } else {
313 /* Handle wait target value node */
314 pend->fence.timestamp = intr->timestamp;
315 complete_all(&pend->fence.completion);
316 }
317 }
318 }
319 spin_unlock(&intr->wait_list_lock);
320
321 if (ts_reg_free_list_head) {
322 INIT_WORK(&job->free_obj, hl_ts_free_objects);
323 job->free_obj_head = ts_reg_free_list_head;
324 job->hdev = hdev;
325 queue_work(hdev->ts_free_obj_wq, &job->free_obj);
326 } else {
327 kfree(job);
328 }
329 }
330
handle_tpc_interrupt(struct hl_device * hdev)331 static void handle_tpc_interrupt(struct hl_device *hdev)
332 {
333 u64 event_mask;
334 u32 flags;
335
336 event_mask = HL_NOTIFIER_EVENT_TPC_ASSERT |
337 HL_NOTIFIER_EVENT_USER_ENGINE_ERR |
338 HL_NOTIFIER_EVENT_DEVICE_RESET;
339
340 flags = HL_DRV_RESET_DELAY;
341
342 dev_err_ratelimited(hdev->dev, "Received TPC assert\n");
343 hl_device_cond_reset(hdev, flags, event_mask);
344 }
345
handle_unexpected_user_interrupt(struct hl_device * hdev)346 static void handle_unexpected_user_interrupt(struct hl_device *hdev)
347 {
348 dev_err_ratelimited(hdev->dev, "Received unexpected user error interrupt\n");
349 }
350
351 /**
352 * hl_irq_handler_user_interrupt - irq handler for user interrupts
353 *
354 * @irq: irq number
355 * @arg: pointer to user interrupt structure
356 *
357 */
hl_irq_handler_user_interrupt(int irq,void * arg)358 irqreturn_t hl_irq_handler_user_interrupt(int irq, void *arg)
359 {
360 struct hl_user_interrupt *user_int = arg;
361
362 user_int->timestamp = ktime_get();
363
364 return IRQ_WAKE_THREAD;
365 }
366
367 /**
368 * hl_irq_user_interrupt_thread_handler - irq thread handler for user interrupts.
369 * This function is invoked by threaded irq mechanism
370 *
371 * @irq: irq number
372 * @arg: pointer to user interrupt structure
373 *
374 */
hl_irq_user_interrupt_thread_handler(int irq,void * arg)375 irqreturn_t hl_irq_user_interrupt_thread_handler(int irq, void *arg)
376 {
377 struct hl_user_interrupt *user_int = arg;
378 struct hl_device *hdev = user_int->hdev;
379
380 switch (user_int->type) {
381 case HL_USR_INTERRUPT_CQ:
382 handle_user_interrupt(hdev, &hdev->common_user_cq_interrupt);
383
384 /* Handle user cq interrupt registered on this specific irq */
385 handle_user_interrupt(hdev, user_int);
386 break;
387 case HL_USR_INTERRUPT_DECODER:
388 handle_user_interrupt(hdev, &hdev->common_decoder_interrupt);
389
390 /* Handle decoder interrupt registered on this specific irq */
391 handle_user_interrupt(hdev, user_int);
392 break;
393 case HL_USR_INTERRUPT_TPC:
394 handle_tpc_interrupt(hdev);
395 break;
396 case HL_USR_INTERRUPT_UNEXPECTED:
397 handle_unexpected_user_interrupt(hdev);
398 break;
399 default:
400 break;
401 }
402
403 return IRQ_HANDLED;
404 }
405
406 /**
407 * hl_irq_handler_eq - irq handler for event queue
408 *
409 * @irq: irq number
410 * @arg: pointer to event queue structure
411 *
412 */
hl_irq_handler_eq(int irq,void * arg)413 irqreturn_t hl_irq_handler_eq(int irq, void *arg)
414 {
415 struct hl_eq *eq = arg;
416 struct hl_device *hdev = eq->hdev;
417 struct hl_eq_entry *eq_entry;
418 struct hl_eq_entry *eq_base;
419 struct hl_eqe_work *handle_eqe_work;
420 bool entry_ready;
421 u32 cur_eqe, ctl;
422 u16 cur_eqe_index, event_type;
423
424 eq_base = eq->kernel_address;
425
426 while (1) {
427 cur_eqe = le32_to_cpu(eq_base[eq->ci].hdr.ctl);
428 entry_ready = !!FIELD_GET(EQ_CTL_READY_MASK, cur_eqe);
429
430 if (!entry_ready)
431 break;
432
433 cur_eqe_index = FIELD_GET(EQ_CTL_INDEX_MASK, cur_eqe);
434 if ((hdev->event_queue.check_eqe_index) &&
435 (((eq->prev_eqe_index + 1) & EQ_CTL_INDEX_MASK) != cur_eqe_index)) {
436 dev_err(hdev->dev,
437 "EQE %#x in queue is ready but index does not match %d!=%d",
438 cur_eqe,
439 ((eq->prev_eqe_index + 1) & EQ_CTL_INDEX_MASK),
440 cur_eqe_index);
441 break;
442 }
443
444 eq->prev_eqe_index++;
445
446 eq_entry = &eq_base[eq->ci];
447
448 /*
449 * Make sure we read EQ entry contents after we've
450 * checked the ownership bit.
451 */
452 dma_rmb();
453
454 if (hdev->disabled && !hdev->reset_info.in_compute_reset) {
455 ctl = le32_to_cpu(eq_entry->hdr.ctl);
456 event_type = ((ctl & EQ_CTL_EVENT_TYPE_MASK) >> EQ_CTL_EVENT_TYPE_SHIFT);
457 dev_warn(hdev->dev,
458 "Device disabled but received an EQ event (%u)\n", event_type);
459 goto skip_irq;
460 }
461
462 handle_eqe_work = kmalloc(sizeof(*handle_eqe_work), GFP_ATOMIC);
463 if (handle_eqe_work) {
464 INIT_WORK(&handle_eqe_work->eq_work, irq_handle_eqe);
465 handle_eqe_work->hdev = hdev;
466
467 memcpy(&handle_eqe_work->eq_entry, eq_entry,
468 sizeof(*eq_entry));
469
470 queue_work(hdev->eq_wq, &handle_eqe_work->eq_work);
471 }
472 skip_irq:
473 /* Clear EQ entry ready bit */
474 eq_entry->hdr.ctl =
475 cpu_to_le32(le32_to_cpu(eq_entry->hdr.ctl) &
476 ~EQ_CTL_READY_MASK);
477
478 eq->ci = hl_eq_inc_ptr(eq->ci);
479
480 hdev->asic_funcs->update_eq_ci(hdev, eq->ci);
481 }
482
483 return IRQ_HANDLED;
484 }
485
486 /**
487 * hl_irq_handler_dec_abnrm - Decoder error interrupt handler
488 * @irq: IRQ number
489 * @arg: pointer to decoder structure.
490 */
hl_irq_handler_dec_abnrm(int irq,void * arg)491 irqreturn_t hl_irq_handler_dec_abnrm(int irq, void *arg)
492 {
493 struct hl_dec *dec = arg;
494
495 schedule_work(&dec->abnrm_intr_work);
496
497 return IRQ_HANDLED;
498 }
499
500 /**
501 * hl_cq_init - main initialization function for an cq object
502 *
503 * @hdev: pointer to device structure
504 * @q: pointer to cq structure
505 * @hw_queue_id: The H/W queue ID this completion queue belongs to
506 * HL_INVALID_QUEUE if cq is not attached to any specific queue
507 *
508 * Allocate dma-able memory for the completion queue and initialize fields
509 * Returns 0 on success
510 */
hl_cq_init(struct hl_device * hdev,struct hl_cq * q,u32 hw_queue_id)511 int hl_cq_init(struct hl_device *hdev, struct hl_cq *q, u32 hw_queue_id)
512 {
513 void *p;
514
515 p = hl_asic_dma_alloc_coherent(hdev, HL_CQ_SIZE_IN_BYTES, &q->bus_address,
516 GFP_KERNEL | __GFP_ZERO);
517 if (!p)
518 return -ENOMEM;
519
520 q->hdev = hdev;
521 q->kernel_address = p;
522 q->hw_queue_id = hw_queue_id;
523 q->ci = 0;
524 q->pi = 0;
525
526 atomic_set(&q->free_slots_cnt, HL_CQ_LENGTH);
527
528 return 0;
529 }
530
531 /**
532 * hl_cq_fini - destroy completion queue
533 *
534 * @hdev: pointer to device structure
535 * @q: pointer to cq structure
536 *
537 * Free the completion queue memory
538 */
hl_cq_fini(struct hl_device * hdev,struct hl_cq * q)539 void hl_cq_fini(struct hl_device *hdev, struct hl_cq *q)
540 {
541 hl_asic_dma_free_coherent(hdev, HL_CQ_SIZE_IN_BYTES, q->kernel_address, q->bus_address);
542 }
543
hl_cq_reset(struct hl_device * hdev,struct hl_cq * q)544 void hl_cq_reset(struct hl_device *hdev, struct hl_cq *q)
545 {
546 q->ci = 0;
547 q->pi = 0;
548
549 atomic_set(&q->free_slots_cnt, HL_CQ_LENGTH);
550
551 /*
552 * It's not enough to just reset the PI/CI because the H/W may have
553 * written valid completion entries before it was halted and therefore
554 * we need to clean the actual queues so we won't process old entries
555 * when the device is operational again
556 */
557
558 memset(q->kernel_address, 0, HL_CQ_SIZE_IN_BYTES);
559 }
560
561 /**
562 * hl_eq_init - main initialization function for an event queue object
563 *
564 * @hdev: pointer to device structure
565 * @q: pointer to eq structure
566 *
567 * Allocate dma-able memory for the event queue and initialize fields
568 * Returns 0 on success
569 */
hl_eq_init(struct hl_device * hdev,struct hl_eq * q)570 int hl_eq_init(struct hl_device *hdev, struct hl_eq *q)
571 {
572 void *p;
573
574 p = hl_cpu_accessible_dma_pool_alloc(hdev, HL_EQ_SIZE_IN_BYTES, &q->bus_address);
575 if (!p)
576 return -ENOMEM;
577
578 q->hdev = hdev;
579 q->kernel_address = p;
580 q->ci = 0;
581 q->prev_eqe_index = 0;
582
583 return 0;
584 }
585
586 /**
587 * hl_eq_fini - destroy event queue
588 *
589 * @hdev: pointer to device structure
590 * @q: pointer to eq structure
591 *
592 * Free the event queue memory
593 */
hl_eq_fini(struct hl_device * hdev,struct hl_eq * q)594 void hl_eq_fini(struct hl_device *hdev, struct hl_eq *q)
595 {
596 flush_workqueue(hdev->eq_wq);
597
598 hl_cpu_accessible_dma_pool_free(hdev, HL_EQ_SIZE_IN_BYTES, q->kernel_address);
599 }
600
hl_eq_reset(struct hl_device * hdev,struct hl_eq * q)601 void hl_eq_reset(struct hl_device *hdev, struct hl_eq *q)
602 {
603 q->ci = 0;
604 q->prev_eqe_index = 0;
605
606 /*
607 * It's not enough to just reset the PI/CI because the H/W may have
608 * written valid completion entries before it was halted and therefore
609 * we need to clean the actual queues so we won't process old entries
610 * when the device is operational again
611 */
612
613 memset(q->kernel_address, 0, HL_EQ_SIZE_IN_BYTES);
614 }
615