Lines Matching refs:xhci
61 static int queue_command(struct xhci_hcd *xhci, struct xhci_command *cmd,
143 static void next_trb(struct xhci_hcd *xhci, in next_trb() argument
159 void inc_deq(struct xhci_hcd *xhci, struct xhci_ring *ring) in inc_deq() argument
179 xhci_warn(xhci, "Missing link TRB at end of segment\n"); in inc_deq()
191 xhci_warn(xhci, "Ring is an endless link TRB loop\n"); in inc_deq()
217 static void inc_enq(struct xhci_hcd *xhci, struct xhci_ring *ring, in inc_enq() argument
230 xhci_err(xhci, "Tried to move enqueue past ring segment\n"); in inc_enq()
254 (xhci->quirks & XHCI_AMD_0x96_HOST)) && in inc_enq()
255 !xhci_link_trb_quirk(xhci)) { in inc_enq()
272 xhci_warn(xhci, "%s: Ring link TRB loop\n", __func__); in inc_enq()
304 static inline int room_on_ring(struct xhci_hcd *xhci, struct xhci_ring *ring, in room_on_ring() argument
322 void xhci_ring_cmd_db(struct xhci_hcd *xhci) in xhci_ring_cmd_db() argument
324 if (!(xhci->cmd_ring_state & CMD_RING_STATE_RUNNING)) in xhci_ring_cmd_db()
327 xhci_dbg(xhci, "// Ding dong!\n"); in xhci_ring_cmd_db()
331 writel(DB_VALUE_HOST, &xhci->dba->doorbell[0]); in xhci_ring_cmd_db()
333 readl(&xhci->dba->doorbell[0]); in xhci_ring_cmd_db()
337 static bool xhci_mod_cmd_timer(struct xhci_hcd *xhci, unsigned long delay) in xhci_mod_cmd_timer() argument
339 return mod_delayed_work(system_wq, &xhci->cmd_timer, delay); in xhci_mod_cmd_timer()
342 static struct xhci_command *xhci_next_queued_cmd(struct xhci_hcd *xhci) in xhci_next_queued_cmd() argument
344 return list_first_entry_or_null(&xhci->cmd_list, struct xhci_command, in xhci_next_queued_cmd()
353 static void xhci_handle_stopped_cmd_ring(struct xhci_hcd *xhci, in xhci_handle_stopped_cmd_ring() argument
359 list_for_each_entry(i_cmd, &xhci->cmd_list, cmd_list) { in xhci_handle_stopped_cmd_ring()
366 xhci_dbg(xhci, "Turn aborted command %p to no-op\n", in xhci_handle_stopped_cmd_ring()
377 xhci->cmd_ring_state = CMD_RING_STATE_RUNNING; in xhci_handle_stopped_cmd_ring()
380 if ((xhci->cmd_ring->dequeue != xhci->cmd_ring->enqueue) && in xhci_handle_stopped_cmd_ring()
381 !(xhci->xhc_state & XHCI_STATE_DYING)) { in xhci_handle_stopped_cmd_ring()
382 xhci->current_cmd = cur_cmd; in xhci_handle_stopped_cmd_ring()
383 xhci_mod_cmd_timer(xhci, XHCI_CMD_DEFAULT_TIMEOUT); in xhci_handle_stopped_cmd_ring()
384 xhci_ring_cmd_db(xhci); in xhci_handle_stopped_cmd_ring()
389 static int xhci_abort_cmd_ring(struct xhci_hcd *xhci, unsigned long flags) in xhci_abort_cmd_ring() argument
391 struct xhci_segment *new_seg = xhci->cmd_ring->deq_seg; in xhci_abort_cmd_ring()
392 union xhci_trb *new_deq = xhci->cmd_ring->dequeue; in xhci_abort_cmd_ring()
396 xhci_dbg(xhci, "Abort command ring\n"); in xhci_abort_cmd_ring()
398 reinit_completion(&xhci->cmd_ring_stop_completion); in xhci_abort_cmd_ring()
408 next_trb(xhci, NULL, &new_seg, &new_deq); in xhci_abort_cmd_ring()
410 next_trb(xhci, NULL, &new_seg, &new_deq); in xhci_abort_cmd_ring()
413 xhci_write_64(xhci, crcr | CMD_RING_ABORT, &xhci->op_regs->cmd_ring); in xhci_abort_cmd_ring()
421 ret = xhci_handshake(&xhci->op_regs->cmd_ring, in xhci_abort_cmd_ring()
424 xhci_err(xhci, "Abort failed to stop command ring: %d\n", ret); in xhci_abort_cmd_ring()
425 xhci_halt(xhci); in xhci_abort_cmd_ring()
426 xhci_hc_died(xhci); in xhci_abort_cmd_ring()
435 spin_unlock_irqrestore(&xhci->lock, flags); in xhci_abort_cmd_ring()
436 ret = wait_for_completion_timeout(&xhci->cmd_ring_stop_completion, in xhci_abort_cmd_ring()
438 spin_lock_irqsave(&xhci->lock, flags); in xhci_abort_cmd_ring()
440 xhci_dbg(xhci, "No stop event for abort, ring start fail?\n"); in xhci_abort_cmd_ring()
441 xhci_cleanup_command_queue(xhci); in xhci_abort_cmd_ring()
443 xhci_handle_stopped_cmd_ring(xhci, xhci_next_queued_cmd(xhci)); in xhci_abort_cmd_ring()
448 void xhci_ring_ep_doorbell(struct xhci_hcd *xhci, in xhci_ring_ep_doorbell() argument
453 __le32 __iomem *db_addr = &xhci->dba->doorbell[slot_id]; in xhci_ring_ep_doorbell()
454 struct xhci_virt_ep *ep = &xhci->devs[slot_id]->eps[ep_index]; in xhci_ring_ep_doorbell()
475 static void ring_doorbell_for_active_rings(struct xhci_hcd *xhci, in ring_doorbell_for_active_rings() argument
482 ep = &xhci->devs[slot_id]->eps[ep_index]; in ring_doorbell_for_active_rings()
487 xhci_ring_ep_doorbell(xhci, slot_id, ep_index, 0); in ring_doorbell_for_active_rings()
495 xhci_ring_ep_doorbell(xhci, slot_id, ep_index, in ring_doorbell_for_active_rings()
500 void xhci_ring_doorbell_for_active_rings(struct xhci_hcd *xhci, in xhci_ring_doorbell_for_active_rings() argument
504 ring_doorbell_for_active_rings(xhci, slot_id, ep_index); in xhci_ring_doorbell_for_active_rings()
507 static struct xhci_virt_ep *xhci_get_virt_ep(struct xhci_hcd *xhci, in xhci_get_virt_ep() argument
512 xhci_warn(xhci, "Invalid slot_id %u\n", slot_id); in xhci_get_virt_ep()
516 xhci_warn(xhci, "Invalid endpoint index %u\n", ep_index); in xhci_get_virt_ep()
519 if (!xhci->devs[slot_id]) { in xhci_get_virt_ep()
520 xhci_warn(xhci, "No xhci virt device for slot_id %u\n", slot_id); in xhci_get_virt_ep()
524 return &xhci->devs[slot_id]->eps[ep_index]; in xhci_get_virt_ep()
527 static struct xhci_ring *xhci_virt_ep_to_ring(struct xhci_hcd *xhci, in xhci_virt_ep_to_ring() argument
539 xhci_warn(xhci, "Invalid stream_id %u request for slot_id %u ep_index %u\n", in xhci_virt_ep_to_ring()
551 struct xhci_ring *xhci_triad_to_transfer_ring(struct xhci_hcd *xhci, in xhci_triad_to_transfer_ring() argument
557 ep = xhci_get_virt_ep(xhci, slot_id, ep_index); in xhci_triad_to_transfer_ring()
561 return xhci_virt_ep_to_ring(xhci, ep, stream_id); in xhci_triad_to_transfer_ring()
571 static u64 xhci_get_hw_deq(struct xhci_hcd *xhci, struct xhci_virt_device *vdev, in xhci_get_hw_deq() argument
584 ep_ctx = xhci_get_ep_ctx(xhci, vdev->out_ctx, ep_index); in xhci_get_hw_deq()
588 static int xhci_move_dequeue_past_td(struct xhci_hcd *xhci, in xhci_move_dequeue_past_td() argument
592 struct xhci_virt_device *dev = xhci->devs[slot_id]; in xhci_move_dequeue_past_td()
609 ep_ring = xhci_triad_to_transfer_ring(xhci, slot_id, in xhci_move_dequeue_past_td()
612 xhci_warn(xhci, "WARN can't find new dequeue, invalid stream ID %u\n", in xhci_move_dequeue_past_td()
628 xhci_dbg(xhci, "ep ring empty, Set new dequeue = enqueue"); in xhci_move_dequeue_past_td()
631 xhci_warn(xhci, "Can't find new dequeue state, missing td\n"); in xhci_move_dequeue_past_td()
636 hw_dequeue = xhci_get_hw_deq(xhci, dev, ep_index, stream_id); in xhci_move_dequeue_past_td()
645 if (xhci->quirks & XHCI_EP_CTX_BROKEN_DCS && in xhci_move_dequeue_past_td()
647 halted_seg = trb_in_td(xhci, td->start_seg, in xhci_move_dequeue_past_td()
655 xhci_dbg(xhci, "Endpoint DCS = %d TRB index = %d cycle = %d\n", in xhci_move_dequeue_past_td()
681 next_trb(xhci, ep_ring, &new_seg, &new_deq); in xhci_move_dequeue_past_td()
685 xhci_err(xhci, "Error: Failed finding new dequeue state\n"); in xhci_move_dequeue_past_td()
696 xhci_warn(xhci, "Can't find dma of new dequeue ptr\n"); in xhci_move_dequeue_past_td()
697 xhci_warn(xhci, "deq seg = %p, deq ptr = %p\n", new_seg, new_deq); in xhci_move_dequeue_past_td()
702 xhci_warn(xhci, "Set TR Deq already pending, don't submit for 0x%pad\n", in xhci_move_dequeue_past_td()
708 cmd = xhci_alloc_command(xhci, false, GFP_ATOMIC); in xhci_move_dequeue_past_td()
710 xhci_warn(xhci, "Can't alloc Set TR Deq cmd 0x%pad\n", &addr); in xhci_move_dequeue_past_td()
716 ret = queue_command(xhci, cmd, in xhci_move_dequeue_past_td()
722 xhci_free_command(xhci, cmd); in xhci_move_dequeue_past_td()
728 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb, in xhci_move_dequeue_past_td()
737 xhci_ring_cmd_db(xhci); in xhci_move_dequeue_past_td()
745 static void td_to_noop(struct xhci_hcd *xhci, struct xhci_ring *ep_ring, in td_to_noop() argument
761 next_trb(xhci, ep_ring, &seg, &trb); in td_to_noop()
765 static void xhci_stop_watchdog_timer_in_irq(struct xhci_hcd *xhci, in xhci_stop_watchdog_timer_in_irq() argument
777 static void xhci_giveback_urb_in_irq(struct xhci_hcd *xhci, in xhci_giveback_urb_in_irq() argument
785 xhci_to_hcd(xhci)->self.bandwidth_isoc_reqs--; in xhci_giveback_urb_in_irq()
786 if (xhci_to_hcd(xhci)->self.bandwidth_isoc_reqs == 0) { in xhci_giveback_urb_in_irq()
787 if (xhci->quirks & XHCI_AMD_PLL_FIX) in xhci_giveback_urb_in_irq()
797 static void xhci_unmap_td_bounce_buffer(struct xhci_hcd *xhci, in xhci_unmap_td_bounce_buffer() argument
800 struct device *dev = xhci_to_hcd(xhci)->self.controller; in xhci_unmap_td_bounce_buffer()
821 xhci_warn(xhci, "WARN Wrong bounce buffer read length: %zu != %d\n", in xhci_unmap_td_bounce_buffer()
831 static int xhci_td_cleanup(struct xhci_hcd *xhci, struct xhci_td *td, in xhci_td_cleanup() argument
840 xhci_unmap_td_bounce_buffer(xhci, ep_ring, td); in xhci_td_cleanup()
848 xhci_warn(xhci, "URB req %u and actual %u transfer length mismatch\n", in xhci_td_cleanup()
866 xhci_dbg(xhci, "Giveback URB %p, len = %d, expected = %d, status = %d\n", in xhci_td_cleanup()
873 xhci_giveback_urb_in_irq(xhci, td, status); in xhci_td_cleanup()
889 ring = xhci_urb_to_transfer_ring(ep->xhci, td->urb); in xhci_giveback_invalidated_tds()
892 xhci_dbg(ep->xhci, "%s: Giveback cancelled URB %p TD\n", in xhci_giveback_invalidated_tds()
894 xhci_td_cleanup(ep->xhci, td, ring, td->status); in xhci_giveback_invalidated_tds()
896 xhci_dbg(ep->xhci, "%s: Keep cancelled URB %p TD as cancel_status is %d\n", in xhci_giveback_invalidated_tds()
899 if (ep->xhci->xhc_state & XHCI_STATE_DYING) in xhci_giveback_invalidated_tds()
904 static int xhci_reset_halted_ep(struct xhci_hcd *xhci, unsigned int slot_id, in xhci_reset_halted_ep() argument
910 command = xhci_alloc_command(xhci, false, GFP_ATOMIC); in xhci_reset_halted_ep()
916 xhci_dbg(xhci, "%s-reset ep %u, slot %u\n", in xhci_reset_halted_ep()
920 ret = xhci_queue_reset_ep(xhci, command, slot_id, ep_index, reset_type); in xhci_reset_halted_ep()
923 xhci_err(xhci, "ERROR queuing reset endpoint for slot %d ep_index %d, %d\n", in xhci_reset_halted_ep()
928 static int xhci_handle_halted_endpoint(struct xhci_hcd *xhci, in xhci_handle_halted_endpoint() argument
953 xhci_dbg(xhci, "Reset ep command for ep_index %d already pending\n", in xhci_handle_halted_endpoint()
958 err = xhci_reset_halted_ep(xhci, slot_id, ep->ep_index, reset_type); in xhci_handle_halted_endpoint()
964 xhci_ring_cmd_db(xhci); in xhci_handle_halted_endpoint()
980 struct xhci_hcd *xhci; in xhci_invalidate_cancelled_tds() local
989 xhci = ep->xhci; in xhci_invalidate_cancelled_tds()
992 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb, in xhci_invalidate_cancelled_tds()
998 ring = xhci_urb_to_transfer_ring(xhci, td->urb); in xhci_invalidate_cancelled_tds()
1000 xhci_warn(xhci, "WARN Cancelled URB %p has invalid stream ID %u.\n", in xhci_invalidate_cancelled_tds()
1010 hw_deq = xhci_get_hw_deq(xhci, ep->vdev, ep->ep_index, in xhci_invalidate_cancelled_tds()
1015 trb_in_td(xhci, td->start_seg, td->first_trb, td->last_trb, hw_deq, false)) { in xhci_invalidate_cancelled_tds()
1025 xhci_dbg(xhci, in xhci_invalidate_cancelled_tds()
1033 td_to_noop(xhci, ring, td, false); in xhci_invalidate_cancelled_tds()
1042 err = xhci_move_dequeue_past_td(xhci, slot_id, ep->ep_index, in xhci_invalidate_cancelled_tds()
1050 xhci_dbg(xhci, "Failed to clear cancelled cached URB %p, mark clear anyway\n", in xhci_invalidate_cancelled_tds()
1052 td_to_noop(xhci, ring, td, false); in xhci_invalidate_cancelled_tds()
1069 hw_deq = xhci_get_hw_deq(ep->xhci, ep->vdev, ep->ep_index, 0); in find_halted_td()
1072 if (trb_in_td(ep->xhci, td->start_seg, td->first_trb, in find_halted_td()
1089 static void xhci_handle_cmd_stop_ep(struct xhci_hcd *xhci, int slot_id, in xhci_handle_cmd_stop_ep() argument
1101 if (!xhci->devs[slot_id]) in xhci_handle_cmd_stop_ep()
1102 xhci_warn(xhci, "Stop endpoint command completion for disabled slot %u\n", in xhci_handle_cmd_stop_ep()
1108 ep = xhci_get_virt_ep(xhci, slot_id, ep_index); in xhci_handle_cmd_stop_ep()
1112 ep_ctx = xhci_get_ep_ctx(xhci, ep->vdev->out_ctx, ep_index); in xhci_handle_cmd_stop_ep()
1133 xhci_dbg(xhci, "Stop ep completion raced with stall, reset ep\n"); in xhci_handle_cmd_stop_ep()
1143 err = xhci_handle_halted_endpoint(xhci, ep, 0, td, in xhci_handle_cmd_stop_ep()
1147 xhci_stop_watchdog_timer_in_irq(xhci, ep); in xhci_handle_cmd_stop_ep()
1151 xhci_dbg(xhci, "Stop ep completion ctx error, ep is running\n"); in xhci_handle_cmd_stop_ep()
1153 command = xhci_alloc_command(xhci, false, GFP_ATOMIC); in xhci_handle_cmd_stop_ep()
1155 xhci_stop_watchdog_timer_in_irq(xhci, ep); in xhci_handle_cmd_stop_ep()
1159 xhci_queue_stop_endpoint(xhci, command, slot_id, ep_index, 0); in xhci_handle_cmd_stop_ep()
1160 xhci_ring_cmd_db(xhci); in xhci_handle_cmd_stop_ep()
1169 xhci_stop_watchdog_timer_in_irq(xhci, ep); in xhci_handle_cmd_stop_ep()
1173 ring_doorbell_for_active_rings(xhci, slot_id, ep_index); in xhci_handle_cmd_stop_ep()
1176 static void xhci_kill_ring_urbs(struct xhci_hcd *xhci, struct xhci_ring *ring) in xhci_kill_ring_urbs() argument
1187 xhci_unmap_td_bounce_buffer(xhci, ring, cur_td); in xhci_kill_ring_urbs()
1191 xhci_giveback_urb_in_irq(xhci, cur_td, -ESHUTDOWN); in xhci_kill_ring_urbs()
1195 static void xhci_kill_endpoint_urbs(struct xhci_hcd *xhci, in xhci_kill_endpoint_urbs() argument
1203 ep = xhci_get_virt_ep(xhci, slot_id, ep_index); in xhci_kill_endpoint_urbs()
1217 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb, in xhci_kill_endpoint_urbs()
1220 xhci_kill_ring_urbs(xhci, ring); in xhci_kill_endpoint_urbs()
1226 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb, in xhci_kill_endpoint_urbs()
1229 xhci_kill_ring_urbs(xhci, ring); in xhci_kill_endpoint_urbs()
1238 xhci_giveback_urb_in_irq(xhci, cur_td, -ESHUTDOWN); in xhci_kill_endpoint_urbs()
1251 void xhci_hc_died(struct xhci_hcd *xhci) in xhci_hc_died() argument
1255 if (xhci->xhc_state & XHCI_STATE_DYING) in xhci_hc_died()
1258 xhci_err(xhci, "xHCI host controller not responding, assume dead\n"); in xhci_hc_died()
1259 xhci->xhc_state |= XHCI_STATE_DYING; in xhci_hc_died()
1261 xhci_cleanup_command_queue(xhci); in xhci_hc_died()
1264 for (i = 0; i <= HCS_MAX_SLOTS(xhci->hcs_params1); i++) { in xhci_hc_died()
1265 if (!xhci->devs[i]) in xhci_hc_died()
1268 xhci_kill_endpoint_urbs(xhci, i, j); in xhci_hc_died()
1272 if (!(xhci->xhc_state & XHCI_STATE_REMOVING)) in xhci_hc_died()
1273 usb_hc_died(xhci_to_hcd(xhci)); in xhci_hc_died()
1296 struct xhci_hcd *xhci = ep->xhci; in xhci_stop_endpoint_command_watchdog() local
1301 spin_lock_irqsave(&xhci->lock, flags); in xhci_stop_endpoint_command_watchdog()
1306 spin_unlock_irqrestore(&xhci->lock, flags); in xhci_stop_endpoint_command_watchdog()
1307 xhci_dbg(xhci, "Stop EP timer raced with cmd completion, exit"); in xhci_stop_endpoint_command_watchdog()
1310 usbsts = readl(&xhci->op_regs->status); in xhci_stop_endpoint_command_watchdog()
1312 xhci_warn(xhci, "xHCI host not responding to stop endpoint command.\n"); in xhci_stop_endpoint_command_watchdog()
1313 xhci_warn(xhci, "USBSTS:%s\n", xhci_decode_usbsts(str, usbsts)); in xhci_stop_endpoint_command_watchdog()
1317 xhci_halt(xhci); in xhci_stop_endpoint_command_watchdog()
1324 xhci_hc_died(xhci); in xhci_stop_endpoint_command_watchdog()
1326 spin_unlock_irqrestore(&xhci->lock, flags); in xhci_stop_endpoint_command_watchdog()
1327 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb, in xhci_stop_endpoint_command_watchdog()
1331 static void update_ring_for_set_deq_completion(struct xhci_hcd *xhci, in update_ring_for_set_deq_completion() argument
1372 xhci_dbg(xhci, "Unable to find new dequeue pointer\n"); in update_ring_for_set_deq_completion()
1384 static void xhci_handle_cmd_set_deq(struct xhci_hcd *xhci, int slot_id, in xhci_handle_cmd_set_deq() argument
1397 ep = xhci_get_virt_ep(xhci, slot_id, ep_index); in xhci_handle_cmd_set_deq()
1401 ep_ring = xhci_virt_ep_to_ring(xhci, ep, stream_id); in xhci_handle_cmd_set_deq()
1403 xhci_warn(xhci, "WARN Set TR deq ptr command for freed stream ID %u\n", in xhci_handle_cmd_set_deq()
1409 ep_ctx = xhci_get_ep_ctx(xhci, ep->vdev->out_ctx, ep_index); in xhci_handle_cmd_set_deq()
1410 slot_ctx = xhci_get_slot_ctx(xhci, ep->vdev->out_ctx); in xhci_handle_cmd_set_deq()
1420 xhci_warn(xhci, "WARN Set TR Deq Ptr cmd invalid because of stream ID configuration\n"); in xhci_handle_cmd_set_deq()
1423 xhci_warn(xhci, "WARN Set TR Deq Ptr cmd failed due to incorrect slot or ep state.\n"); in xhci_handle_cmd_set_deq()
1427 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb, in xhci_handle_cmd_set_deq()
1432 xhci_warn(xhci, "WARN Set TR Deq Ptr cmd failed because slot %u was not enabled.\n", in xhci_handle_cmd_set_deq()
1436 xhci_warn(xhci, "WARN Set TR Deq Ptr cmd with unknown completion code of %u.\n", in xhci_handle_cmd_set_deq()
1456 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb, in xhci_handle_cmd_set_deq()
1463 update_ring_for_set_deq_completion(xhci, ep->vdev, in xhci_handle_cmd_set_deq()
1466 xhci_warn(xhci, "Mismatch between completed Set TR Deq Ptr command & xHCI internal state.\n"); in xhci_handle_cmd_set_deq()
1467 xhci_warn(xhci, "ep deq seg = %p, deq ptr = %p\n", in xhci_handle_cmd_set_deq()
1474 ep_ring = xhci_urb_to_transfer_ring(ep->xhci, td->urb); in xhci_handle_cmd_set_deq()
1477 xhci_dbg(ep->xhci, "%s: Giveback cancelled URB %p TD\n", in xhci_handle_cmd_set_deq()
1479 xhci_td_cleanup(ep->xhci, td, ep_ring, td->status); in xhci_handle_cmd_set_deq()
1481 xhci_dbg(ep->xhci, "%s: Keep cancelled URB %p TD as cancel_status is %d\n", in xhci_handle_cmd_set_deq()
1490 ring_doorbell_for_active_rings(xhci, slot_id, ep_index); in xhci_handle_cmd_set_deq()
1493 static void xhci_handle_cmd_reset_ep(struct xhci_hcd *xhci, int slot_id, in xhci_handle_cmd_reset_ep() argument
1501 ep = xhci_get_virt_ep(xhci, slot_id, ep_index); in xhci_handle_cmd_reset_ep()
1505 ep_ctx = xhci_get_ep_ctx(xhci, ep->vdev->out_ctx, ep_index); in xhci_handle_cmd_reset_ep()
1511 xhci_dbg_trace(xhci, trace_xhci_dbg_reset_ep, in xhci_handle_cmd_reset_ep()
1517 if (xhci->quirks & XHCI_RESET_EP_QUIRK) in xhci_handle_cmd_reset_ep()
1518 xhci_dbg(xhci, "Note: Removed workaround to queue config ep for this hw"); in xhci_handle_cmd_reset_ep()
1526 ring_doorbell_for_active_rings(xhci, slot_id, ep_index); in xhci_handle_cmd_reset_ep()
1529 static void xhci_handle_cmd_enable_slot(struct xhci_hcd *xhci, int slot_id, in xhci_handle_cmd_enable_slot() argument
1538 static void xhci_handle_cmd_disable_slot(struct xhci_hcd *xhci, int slot_id) in xhci_handle_cmd_disable_slot() argument
1543 virt_dev = xhci->devs[slot_id]; in xhci_handle_cmd_disable_slot()
1547 slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->out_ctx); in xhci_handle_cmd_disable_slot()
1550 if (xhci->quirks & XHCI_EP_LIMIT_QUIRK) in xhci_handle_cmd_disable_slot()
1552 xhci_free_device_endpoint_resources(xhci, virt_dev, true); in xhci_handle_cmd_disable_slot()
1555 static void xhci_handle_cmd_config_ep(struct xhci_hcd *xhci, int slot_id, in xhci_handle_cmd_config_ep() argument
1573 virt_dev = xhci->devs[slot_id]; in xhci_handle_cmd_config_ep()
1578 xhci_warn(xhci, "Could not get input context, bad type.\n"); in xhci_handle_cmd_config_ep()
1587 ep_ctx = xhci_get_ep_ctx(xhci, virt_dev->out_ctx, ep_index); in xhci_handle_cmd_config_ep()
1596 if (xhci->quirks & XHCI_RESET_EP_QUIRK && in xhci_handle_cmd_config_ep()
1602 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks, in xhci_handle_cmd_config_ep()
1608 ring_doorbell_for_active_rings(xhci, slot_id, ep_index); in xhci_handle_cmd_config_ep()
1614 static void xhci_handle_cmd_addr_dev(struct xhci_hcd *xhci, int slot_id) in xhci_handle_cmd_addr_dev() argument
1619 vdev = xhci->devs[slot_id]; in xhci_handle_cmd_addr_dev()
1622 slot_ctx = xhci_get_slot_ctx(xhci, vdev->out_ctx); in xhci_handle_cmd_addr_dev()
1626 static void xhci_handle_cmd_reset_dev(struct xhci_hcd *xhci, int slot_id) in xhci_handle_cmd_reset_dev() argument
1631 vdev = xhci->devs[slot_id]; in xhci_handle_cmd_reset_dev()
1633 xhci_warn(xhci, "Reset device command completion for disabled slot %u\n", in xhci_handle_cmd_reset_dev()
1637 slot_ctx = xhci_get_slot_ctx(xhci, vdev->out_ctx); in xhci_handle_cmd_reset_dev()
1640 xhci_dbg(xhci, "Completed reset device command.\n"); in xhci_handle_cmd_reset_dev()
1643 static void xhci_handle_cmd_nec_get_fw(struct xhci_hcd *xhci, in xhci_handle_cmd_nec_get_fw() argument
1646 if (!(xhci->quirks & XHCI_NEC_HOST)) { in xhci_handle_cmd_nec_get_fw()
1647 xhci_warn(xhci, "WARN NEC_GET_FW command on non-NEC host\n"); in xhci_handle_cmd_nec_get_fw()
1650 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks, in xhci_handle_cmd_nec_get_fw()
1668 void xhci_cleanup_command_queue(struct xhci_hcd *xhci) in xhci_cleanup_command_queue() argument
1671 xhci->current_cmd = NULL; in xhci_cleanup_command_queue()
1672 list_for_each_entry_safe(cur_cmd, tmp_cmd, &xhci->cmd_list, cmd_list) in xhci_cleanup_command_queue()
1678 struct xhci_hcd *xhci; in xhci_handle_command_timeout() local
1682 xhci = container_of(to_delayed_work(work), struct xhci_hcd, cmd_timer); in xhci_handle_command_timeout()
1684 spin_lock_irqsave(&xhci->lock, flags); in xhci_handle_command_timeout()
1690 if (!xhci->current_cmd || delayed_work_pending(&xhci->cmd_timer)) { in xhci_handle_command_timeout()
1691 spin_unlock_irqrestore(&xhci->lock, flags); in xhci_handle_command_timeout()
1695 xhci->current_cmd->status = COMP_COMMAND_ABORTED; in xhci_handle_command_timeout()
1698 hw_ring_state = xhci_read_64(xhci, &xhci->op_regs->cmd_ring); in xhci_handle_command_timeout()
1700 xhci_hc_died(xhci); in xhci_handle_command_timeout()
1704 if ((xhci->cmd_ring_state & CMD_RING_STATE_RUNNING) && in xhci_handle_command_timeout()
1707 xhci->cmd_ring_state = CMD_RING_STATE_ABORTED; in xhci_handle_command_timeout()
1708 xhci_dbg(xhci, "Command timeout\n"); in xhci_handle_command_timeout()
1709 xhci_abort_cmd_ring(xhci, flags); in xhci_handle_command_timeout()
1714 if (xhci->xhc_state & XHCI_STATE_REMOVING) { in xhci_handle_command_timeout()
1715 xhci_dbg(xhci, "host removed, ring start fail?\n"); in xhci_handle_command_timeout()
1716 xhci_cleanup_command_queue(xhci); in xhci_handle_command_timeout()
1722 xhci_dbg(xhci, "Command timeout on stopped ring\n"); in xhci_handle_command_timeout()
1723 xhci_handle_stopped_cmd_ring(xhci, xhci->current_cmd); in xhci_handle_command_timeout()
1726 spin_unlock_irqrestore(&xhci->lock, flags); in xhci_handle_command_timeout()
1730 static void handle_cmd_completion(struct xhci_hcd *xhci, in handle_cmd_completion() argument
1742 xhci_warn(xhci, "Invalid slot_id %u\n", slot_id); in handle_cmd_completion()
1747 cmd_trb = xhci->cmd_ring->dequeue; in handle_cmd_completion()
1749 trace_xhci_handle_command(xhci->cmd_ring, &cmd_trb->generic); in handle_cmd_completion()
1751 cmd_dequeue_dma = xhci_trb_virt_to_dma(xhci->cmd_ring->deq_seg, in handle_cmd_completion()
1758 xhci_warn(xhci, in handle_cmd_completion()
1763 cmd = list_first_entry(&xhci->cmd_list, struct xhci_command, cmd_list); in handle_cmd_completion()
1765 cancel_delayed_work(&xhci->cmd_timer); in handle_cmd_completion()
1771 complete_all(&xhci->cmd_ring_stop_completion); in handle_cmd_completion()
1775 if (cmd->command_trb != xhci->cmd_ring->dequeue) { in handle_cmd_completion()
1776 xhci_err(xhci, in handle_cmd_completion()
1788 xhci->cmd_ring_state = CMD_RING_STATE_STOPPED; in handle_cmd_completion()
1790 if (xhci->current_cmd == cmd) in handle_cmd_completion()
1791 xhci->current_cmd = NULL; in handle_cmd_completion()
1799 xhci_handle_cmd_enable_slot(xhci, slot_id, cmd, cmd_comp_code); in handle_cmd_completion()
1802 xhci_handle_cmd_disable_slot(xhci, slot_id); in handle_cmd_completion()
1806 xhci_handle_cmd_config_ep(xhci, slot_id, cmd_comp_code); in handle_cmd_completion()
1811 xhci_handle_cmd_addr_dev(xhci, slot_id); in handle_cmd_completion()
1817 xhci_handle_cmd_stop_ep(xhci, slot_id, cmd_trb, in handle_cmd_completion()
1823 xhci_handle_cmd_set_deq(xhci, slot_id, cmd_trb, cmd_comp_code); in handle_cmd_completion()
1833 xhci_handle_cmd_reset_ep(xhci, slot_id, cmd_trb, cmd_comp_code); in handle_cmd_completion()
1841 xhci_handle_cmd_reset_dev(xhci, slot_id); in handle_cmd_completion()
1844 xhci_handle_cmd_nec_get_fw(xhci, event); in handle_cmd_completion()
1848 xhci_info(xhci, "INFO unknown command type %d\n", cmd_type); in handle_cmd_completion()
1853 if (!list_is_singular(&xhci->cmd_list)) { in handle_cmd_completion()
1854 xhci->current_cmd = list_first_entry(&cmd->cmd_list, in handle_cmd_completion()
1856 xhci_mod_cmd_timer(xhci, XHCI_CMD_DEFAULT_TIMEOUT); in handle_cmd_completion()
1857 } else if (xhci->current_cmd == cmd) { in handle_cmd_completion()
1858 xhci->current_cmd = NULL; in handle_cmd_completion()
1864 inc_deq(xhci, xhci->cmd_ring); in handle_cmd_completion()
1867 static void handle_vendor_event(struct xhci_hcd *xhci, in handle_vendor_event() argument
1870 xhci_dbg(xhci, "Vendor specific event TRB type = %u\n", trb_type); in handle_vendor_event()
1871 if (trb_type == TRB_NEC_CMD_COMP && (xhci->quirks & XHCI_NEC_HOST)) in handle_vendor_event()
1872 handle_cmd_completion(xhci, &event->event_cmd); in handle_vendor_event()
1875 static void handle_device_notification(struct xhci_hcd *xhci, in handle_device_notification() argument
1882 if (!xhci->devs[slot_id]) { in handle_device_notification()
1883 xhci_warn(xhci, "Device Notification event for " in handle_device_notification()
1888 xhci_dbg(xhci, "Device Wake Notification event for slot ID %u\n", in handle_device_notification()
1890 udev = xhci->devs[slot_id]->udev; in handle_device_notification()
1907 static void xhci_cavium_reset_phy_quirk(struct xhci_hcd *xhci) in xhci_cavium_reset_phy_quirk() argument
1909 struct usb_hcd *hcd = xhci_to_hcd(xhci); in xhci_cavium_reset_phy_quirk()
1924 static void handle_port_status(struct xhci_hcd *xhci, in handle_port_status() argument
1939 xhci_warn(xhci, in handle_port_status()
1943 max_ports = HCS_MAX_PORTS(xhci->hcs_params1); in handle_port_status()
1946 xhci_warn(xhci, "Port change event with invalid port ID %d\n", in handle_port_status()
1948 inc_deq(xhci, xhci->event_ring); in handle_port_status()
1952 port = &xhci->hw_ports[port_id - 1]; in handle_port_status()
1954 xhci_warn(xhci, "Port change event, no port for port ID %u\n", in handle_port_status()
1961 if (port->rhub == &xhci->usb3_rhub && xhci->shared_hcd == NULL) { in handle_port_status()
1962 xhci_dbg(xhci, "ignore port event for removed USB3 hcd\n"); in handle_port_status()
1972 xhci_dbg(xhci, "Port change event, %d-%d, id %d, portsc: 0x%x\n", in handle_port_status()
1978 xhci_dbg(xhci, "resume root hub\n"); in handle_port_status()
1984 slot_id = xhci_find_slot_id_by_port(hcd, xhci, hcd_portnum + 1); in handle_port_status()
1985 if (slot_id && xhci->devs[slot_id]) in handle_port_status()
1986 xhci->devs[slot_id]->flags |= VDEV_PORT_ERROR; in handle_port_status()
1990 xhci_dbg(xhci, "port resume event for port %d\n", port_id); in handle_port_status()
1992 cmd_reg = readl(&xhci->op_regs->command); in handle_port_status()
1994 xhci_warn(xhci, "xHC is not running.\n"); in handle_port_status()
1999 xhci_dbg(xhci, "remote wake SS port %d\n", port_id); in handle_port_status()
2005 xhci_test_and_clear_bit(xhci, port, PORT_PLC); in handle_port_status()
2007 xhci_set_link_state(xhci, port, XDEV_U0); in handle_port_status()
2014 xhci_dbg(xhci, "resume HS port %d\n", port_id); in handle_port_status()
2035 xhci_dbg(xhci, "resume SS port %d finished\n", port_id); in handle_port_status()
2044 slot_id = xhci_find_slot_id_by_port(hcd, xhci, hcd_portnum + 1); in handle_port_status()
2045 if (slot_id && xhci->devs[slot_id]) in handle_port_status()
2046 xhci_ring_device(xhci, slot_id); in handle_port_status()
2048 xhci_test_and_clear_bit(xhci, port, PORT_PLC); in handle_port_status()
2070 xhci_test_and_clear_bit(xhci, port, PORT_PLC); in handle_port_status()
2071 if ((xhci->quirks & XHCI_RESET_PLL_ON_DISCONNECT) && in handle_port_status()
2073 xhci_cavium_reset_phy_quirk(xhci); in handle_port_status()
2078 inc_deq(xhci, xhci->event_ring); in handle_port_status()
2094 xhci_dbg(xhci, "%s: starting usb%d port polling.\n", in handle_port_status()
2097 spin_unlock(&xhci->lock); in handle_port_status()
2100 spin_lock(&xhci->lock); in handle_port_status()
2109 struct xhci_segment *trb_in_td(struct xhci_hcd *xhci, in trb_in_td() argument
2134 xhci_warn(xhci, in trb_in_td()
2170 static void xhci_clear_hub_tt_buffer(struct xhci_hcd *xhci, struct xhci_td *td, in xhci_clear_hub_tt_buffer() argument
2178 (td->urb->dev->tt->hub != xhci_to_hcd(xhci)->self.root_hub) && in xhci_clear_hub_tt_buffer()
2193 static int xhci_requires_manual_halt_cleanup(struct xhci_hcd *xhci, in xhci_requires_manual_halt_cleanup() argument
2213 int xhci_is_vendor_info_code(struct xhci_hcd *xhci, unsigned int trb_comp_code) in xhci_is_vendor_info_code() argument
2219 xhci_dbg(xhci, "Vendor defined info completion code %u\n", in xhci_is_vendor_info_code()
2221 xhci_dbg(xhci, "Treating code as success.\n"); in xhci_is_vendor_info_code()
2227 static int finish_td(struct xhci_hcd *xhci, struct xhci_virt_ep *ep, in finish_td() argument
2234 ep_ctx = xhci_get_ep_ctx(xhci, ep->vdev->out_ctx, ep->ep_index); in finish_td()
2268 xhci_dbg(xhci, "Already resolving halted ep for 0x%llx\n", in finish_td()
2277 xhci_clear_hub_tt_buffer(xhci, td, ep); in finish_td()
2278 xhci_handle_halted_endpoint(xhci, ep, ep_ring->stream_id, td, in finish_td()
2293 xhci_clear_hub_tt_buffer(xhci, td, ep); in finish_td()
2295 xhci_handle_halted_endpoint(xhci, ep, ep_ring->stream_id, td, in finish_td()
2308 xhci_dbg(xhci, "Failed to count freed trbs at TD finish\n"); in finish_td()
2313 inc_deq(xhci, ep_ring); in finish_td()
2315 return xhci_td_cleanup(xhci, td, ep_ring, td->status); in finish_td()
2319 static int sum_trb_lengths(struct xhci_hcd *xhci, struct xhci_ring *ring, in sum_trb_lengths() argument
2326 for (sum = 0; trb != stop_trb; next_trb(xhci, ring, &seg, &trb)) { in sum_trb_lengths()
2336 static int process_ctrl_td(struct xhci_hcd *xhci, struct xhci_virt_ep *ep, in process_ctrl_td() argument
2346 ep_ctx = xhci_get_ep_ctx(xhci, ep->vdev->out_ctx, ep->ep_index); in process_ctrl_td()
2354 xhci_warn(xhci, "WARN: Success on ctrl %s TRB without IOC set?\n", in process_ctrl_td()
2368 xhci_warn(xhci, "WARN: Stopped Short Packet on ctrl setup or status TRB\n"); in process_ctrl_td()
2383 xhci_warn(xhci, "WARN: unexpected TRB Type %d\n", in process_ctrl_td()
2390 if (!xhci_requires_manual_halt_cleanup(xhci, in process_ctrl_td()
2393 xhci_dbg(xhci, "TRB error %u, halted endpoint index = %u\n", in process_ctrl_td()
2417 xhci_dbg(xhci, "Waiting for status stage event\n"); in process_ctrl_td()
2426 return finish_td(xhci, ep, ep_ring, td, trb_comp_code); in process_ctrl_td()
2432 static int process_isoc_td(struct xhci_hcd *xhci, struct xhci_virt_ep *ep, in process_isoc_td() argument
2462 if (xhci->quirks & XHCI_TRUST_TX_LENGTH) in process_isoc_td()
2515 frame->actual_length = sum_trb_lengths(xhci, ep->ring, ep_trb) + in process_isoc_td()
2525 xhci_dbg(xhci, "Error mid isoc TD, wait for final completion event\n"); in process_isoc_td()
2530 return finish_td(xhci, ep, ep_ring, td, trb_comp_code); in process_isoc_td()
2533 static int skip_isoc_td(struct xhci_hcd *xhci, struct xhci_td *td, in skip_isoc_td() argument
2554 inc_deq(xhci, ep->ring); in skip_isoc_td()
2556 return xhci_td_cleanup(xhci, td, ep->ring, status); in skip_isoc_td()
2562 static int process_bulk_intr_td(struct xhci_hcd *xhci, struct xhci_virt_ep *ep, in process_bulk_intr_td() argument
2570 slot_ctx = xhci_get_slot_ctx(xhci, ep->vdev->out_ctx); in process_bulk_intr_td()
2581 xhci_warn(xhci, "WARN Successful completion on short TX\n"); in process_bulk_intr_td()
2582 xhci_dbg(xhci, "ep %#x - asked for %d bytes, %d bytes untransferred\n", in process_bulk_intr_td()
2589 xhci_dbg(xhci, "ep %#x - asked for %d bytes, %d bytes untransferred\n", in process_bulk_intr_td()
2603 if (xhci->quirks & XHCI_NO_SOFT_RETRY || in process_bulk_intr_td()
2610 xhci_handle_halted_endpoint(xhci, ep, ep_ring->stream_id, td, in process_bulk_intr_td()
2622 sum_trb_lengths(xhci, ep_ring, ep_trb) + in process_bulk_intr_td()
2626 xhci_warn(xhci, "bad transfer trb length %d in event trb\n", in process_bulk_intr_td()
2631 return finish_td(xhci, ep, ep_ring, td, trb_comp_code); in process_bulk_intr_td()
2639 static int handle_tx_event(struct xhci_hcd *xhci, in handle_tx_event() argument
2662 ep = xhci_get_virt_ep(xhci, slot_id, ep_index); in handle_tx_event()
2664 xhci_err(xhci, "ERROR Invalid Transfer event\n"); in handle_tx_event()
2669 ep_ctx = xhci_get_ep_ctx(xhci, ep->vdev->out_ctx, ep_index); in handle_tx_event()
2672 xhci_err(xhci, in handle_tx_event()
2685 xhci_handle_halted_endpoint(xhci, ep, 0, NULL, in handle_tx_event()
2693 xhci_err(xhci, "ERROR Transfer event for unknown stream ring slot %u ep %u\n", in handle_tx_event()
2713 if (xhci->quirks & XHCI_TRUST_TX_LENGTH || in handle_tx_event()
2717 xhci_warn_ratelimited(xhci, in handle_tx_event()
2724 xhci_dbg(xhci, "Stopped on Transfer TRB for slot %u ep %u\n", in handle_tx_event()
2728 xhci_dbg(xhci, in handle_tx_event()
2733 xhci_dbg(xhci, in handle_tx_event()
2739 xhci_dbg(xhci, "Stalled endpoint for slot %u ep %u\n", slot_id, in handle_tx_event()
2744 xhci_dbg(xhci, "Split transaction error for slot %u ep %u\n", in handle_tx_event()
2749 xhci_dbg(xhci, "Transfer error for slot %u ep %u on endpoint\n", in handle_tx_event()
2754 xhci_dbg(xhci, "Babble error for slot %u ep %u on endpoint\n", in handle_tx_event()
2760 xhci_warn(xhci, in handle_tx_event()
2767 xhci_warn(xhci, in handle_tx_event()
2773 xhci_warn(xhci, in handle_tx_event()
2778 xhci_warn(xhci, in handle_tx_event()
2788 xhci_dbg(xhci, "underrun event on endpoint\n"); in handle_tx_event()
2790 xhci_dbg(xhci, "Underrun Event for slot %d ep %d " in handle_tx_event()
2796 xhci_dbg(xhci, "overrun event on endpoint\n"); in handle_tx_event()
2798 xhci_dbg(xhci, "Overrun Event for slot %d ep %d " in handle_tx_event()
2811 xhci_dbg(xhci, in handle_tx_event()
2817 xhci_dbg(xhci, in handle_tx_event()
2824 xhci_warn(xhci, in handle_tx_event()
2830 if (xhci_is_vendor_info_code(xhci, trb_comp_code)) { in handle_tx_event()
2834 xhci_warn(xhci, in handle_tx_event()
2856 xhci_warn(xhci, "WARN Event TRB for slot %d ep %d with no TDs queued?\n", in handle_tx_event()
2862 xhci_dbg(xhci, "td_list is empty while skip flag set. Clear skip flag for slot %u ep %u.\n", in handle_tx_event()
2866 xhci_requires_manual_halt_cleanup(xhci, ep_ctx, in handle_tx_event()
2868 xhci_handle_halted_endpoint(xhci, ep, in handle_tx_event()
2879 xhci_dbg(xhci, "All tds on the ep_ring skipped. Clear skip flag for slot %u ep %u.\n", in handle_tx_event()
2890 ep_seg = trb_in_td(xhci, ep_ring->deq_seg, ep_ring->dequeue, in handle_tx_event()
2909 skip_isoc_td(xhci, td, ep, status); in handle_tx_event()
2917 if ((xhci->quirks & XHCI_SPURIOUS_SUCCESS) && in handle_tx_event()
2938 ep_seg = trb_in_td(xhci, td_next->start_seg, td_next->first_trb, in handle_tx_event()
2942 xhci_dbg(xhci, "Missing TD completion event after mid TD error\n"); in handle_tx_event()
2945 inc_deq(xhci, ep_ring); in handle_tx_event()
2946 xhci_td_cleanup(xhci, td, ep_ring, td->status); in handle_tx_event()
2953 xhci_err(xhci, in handle_tx_event()
2958 trb_in_td(xhci, ep_ring->deq_seg, in handle_tx_event()
2970 xhci_dbg(xhci, in handle_tx_event()
2992 xhci_requires_manual_halt_cleanup(xhci, ep_ctx, in handle_tx_event()
2994 xhci_handle_halted_endpoint(xhci, ep, in handle_tx_event()
3004 process_ctrl_td(xhci, ep, ep_ring, td, ep_trb, event); in handle_tx_event()
3006 process_isoc_td(xhci, ep, ep_ring, td, ep_trb, event); in handle_tx_event()
3008 process_bulk_intr_td(xhci, ep, ep_ring, td, ep_trb, event); in handle_tx_event()
3019 inc_deq(xhci, xhci->event_ring); in handle_tx_event()
3032 xhci_err(xhci, "@%016llx %08x %08x %08x %08x\n", in handle_tx_event()
3034 xhci->event_ring->deq_seg, in handle_tx_event()
3035 xhci->event_ring->dequeue), in handle_tx_event()
3049 int xhci_handle_event(struct xhci_hcd *xhci) in xhci_handle_event() argument
3057 if (!xhci->event_ring || !xhci->event_ring->dequeue) { in xhci_handle_event()
3058 xhci_err(xhci, "ERROR event ring not ready\n"); in xhci_handle_event()
3062 event = xhci->event_ring->dequeue; in xhci_handle_event()
3065 xhci->event_ring->cycle_state) in xhci_handle_event()
3068 trace_xhci_handle_event(xhci->event_ring, &event->generic); in xhci_handle_event()
3080 handle_cmd_completion(xhci, &event->event_cmd); in xhci_handle_event()
3083 handle_port_status(xhci, event); in xhci_handle_event()
3087 ret = handle_tx_event(xhci, &event->trans_event); in xhci_handle_event()
3092 handle_device_notification(xhci, event); in xhci_handle_event()
3096 handle_vendor_event(xhci, event, trb_type); in xhci_handle_event()
3098 xhci_warn(xhci, "ERROR unknown event type %d\n", trb_type); in xhci_handle_event()
3103 if (xhci->xhc_state & XHCI_STATE_DYING) { in xhci_handle_event()
3104 xhci_dbg(xhci, "xHCI host dying, returning from " in xhci_handle_event()
3111 inc_deq(xhci, xhci->event_ring); in xhci_handle_event()
3125 void xhci_update_erst_dequeue(struct xhci_hcd *xhci, in xhci_update_erst_dequeue() argument
3131 temp_64 = xhci_read_64(xhci, &xhci->ir_set->erst_dequeue); in xhci_update_erst_dequeue()
3133 if (event_ring_deq != xhci->event_ring->dequeue) { in xhci_update_erst_dequeue()
3134 deq = xhci_trb_virt_to_dma(xhci->event_ring->deq_seg, in xhci_update_erst_dequeue()
3135 xhci->event_ring->dequeue); in xhci_update_erst_dequeue()
3137 xhci_warn(xhci, "WARN something wrong with SW event ring dequeue ptr\n"); in xhci_update_erst_dequeue()
3153 xhci_write_64(xhci, temp_64, &xhci->ir_set->erst_dequeue); in xhci_update_erst_dequeue()
3157 static irqreturn_t xhci_vendor_queue_irq_work(struct xhci_hcd *xhci) in xhci_vendor_queue_irq_work() argument
3159 struct xhci_vendor_ops *ops = xhci_vendor_get_ops(xhci); in xhci_vendor_queue_irq_work()
3162 return ops->queue_irq_work(xhci); in xhci_vendor_queue_irq_work()
3173 struct xhci_hcd *xhci = hcd_to_xhci(hcd); in xhci_irq() local
3181 spin_lock_irqsave(&xhci->lock, flags); in xhci_irq()
3183 status = readl(&xhci->op_regs->status); in xhci_irq()
3185 xhci_hc_died(xhci); in xhci_irq()
3194 xhci_warn(xhci, "WARNING: Host System Error\n"); in xhci_irq()
3195 xhci_halt(xhci); in xhci_irq()
3200 ret = xhci_vendor_queue_irq_work(xhci); in xhci_irq()
3210 writel(status, &xhci->op_regs->status); in xhci_irq()
3214 irq_pending = readl(&xhci->ir_set->irq_pending); in xhci_irq()
3216 writel(irq_pending, &xhci->ir_set->irq_pending); in xhci_irq()
3219 if (xhci->xhc_state & XHCI_STATE_DYING || in xhci_irq()
3220 xhci->xhc_state & XHCI_STATE_HALTED) { in xhci_irq()
3221 xhci_dbg(xhci, "xHCI dying, ignoring interrupt. " in xhci_irq()
3226 temp_64 = xhci_read_64(xhci, &xhci->ir_set->erst_dequeue); in xhci_irq()
3227 xhci_write_64(xhci, temp_64 | ERST_EHB, in xhci_irq()
3228 &xhci->ir_set->erst_dequeue); in xhci_irq()
3233 event_ring_deq = xhci->event_ring->dequeue; in xhci_irq()
3237 while (xhci_handle_event(xhci) > 0) { in xhci_irq()
3240 xhci_update_erst_dequeue(xhci, event_ring_deq); in xhci_irq()
3241 event_ring_deq = xhci->event_ring->dequeue; in xhci_irq()
3246 xhci_update_erst_dequeue(xhci, event_ring_deq); in xhci_irq()
3250 spin_unlock_irqrestore(&xhci->lock, flags); in xhci_irq()
3269 static void queue_trb(struct xhci_hcd *xhci, struct xhci_ring *ring, in queue_trb() argument
3285 inc_enq(xhci, ring, more_trbs_coming); in queue_trb()
3292 static int prepare_ring(struct xhci_hcd *xhci, struct xhci_ring *ep_ring, in prepare_ring() argument
3305 xhci_warn(xhci, "WARN urb submitted to disabled ep\n"); in prepare_ring()
3308 xhci_warn(xhci, "WARN waiting for error on ep to be cleared\n"); in prepare_ring()
3313 xhci_dbg(xhci, "WARN halted endpoint, queueing URB anyway.\n"); in prepare_ring()
3318 xhci_err(xhci, "ERROR unknown endpoint state for ep\n"); in prepare_ring()
3327 if (room_on_ring(xhci, ep_ring, num_trbs)) in prepare_ring()
3330 if (ep_ring == xhci->cmd_ring) { in prepare_ring()
3331 xhci_err(xhci, "Do not support expand command ring\n"); in prepare_ring()
3335 xhci_dbg_trace(xhci, trace_xhci_dbg_ring_expansion, in prepare_ring()
3338 if (xhci_ring_expansion(xhci, ep_ring, num_trbs_needed, in prepare_ring()
3340 xhci_err(xhci, "Ring expansion failed\n"); in prepare_ring()
3349 if (!xhci_link_trb_quirk(xhci) && in prepare_ring()
3351 (xhci->quirks & XHCI_AMD_0x96_HOST))) in prepare_ring()
3370 xhci_warn(xhci, "Ring is an endless link TRB loop\n"); in prepare_ring()
3376 xhci_warn(xhci, "Missing link TRB at end of ring segment\n"); in prepare_ring()
3383 static int prepare_transfer(struct xhci_hcd *xhci, in prepare_transfer() argument
3396 struct xhci_ep_ctx *ep_ctx = xhci_get_ep_ctx(xhci, xdev->out_ctx, ep_index); in prepare_transfer()
3398 ep_ring = xhci_triad_to_transfer_ring(xhci, xdev->slot_id, ep_index, in prepare_transfer()
3401 xhci_dbg(xhci, "Can't prepare ring for bad stream ID %u\n", in prepare_transfer()
3406 ret = prepare_ring(xhci, ep_ring, GET_EP_CTX_STATE(ep_ctx), in prepare_transfer()
3490 static void giveback_first_trb(struct xhci_hcd *xhci, int slot_id, in giveback_first_trb() argument
3503 xhci_ring_ep_doorbell(xhci, slot_id, ep_index, stream_id); in giveback_first_trb()
3506 static void check_interval(struct xhci_hcd *xhci, struct urb *urb, in check_interval() argument
3542 int xhci_queue_intr_tx(struct xhci_hcd *xhci, gfp_t mem_flags, in xhci_queue_intr_tx() argument
3547 ep_ctx = xhci_get_ep_ctx(xhci, xhci->devs[slot_id]->out_ctx, ep_index); in xhci_queue_intr_tx()
3548 check_interval(xhci, urb, ep_ctx); in xhci_queue_intr_tx()
3550 return xhci_queue_bulk_tx(xhci, mem_flags, urb, slot_id, ep_index); in xhci_queue_intr_tx()
3573 static u32 xhci_td_remainder(struct xhci_hcd *xhci, int transferred, in xhci_td_remainder() argument
3580 if (xhci->hci_version < 0x100 && !(xhci->quirks & XHCI_MTK_HOST)) in xhci_td_remainder()
3589 if ((xhci->quirks & XHCI_MTK_HOST) && (xhci->hci_version < 0x100)) in xhci_td_remainder()
3600 static int xhci_align_td(struct xhci_hcd *xhci, struct urb *urb, u32 enqd_len, in xhci_align_td() argument
3603 struct device *dev = xhci_to_hcd(xhci)->self.controller; in xhci_align_td()
3616 xhci_dbg(xhci, "Unaligned %d bytes, buff len %d\n", in xhci_align_td()
3622 xhci_dbg(xhci, "split align, new buff len %d\n", *trb_buff_len); in xhci_align_td()
3642 xhci_warn(xhci, "WARN Wrong bounce buffer write length: %zu != %d\n", in xhci_align_td()
3657 xhci_warn(xhci, "Failed mapping bounce buffer, not aligning\n"); in xhci_align_td()
3664 xhci_dbg(xhci, "Bounce align, new buff len %d\n", *trb_buff_len); in xhci_align_td()
3670 int xhci_queue_bulk_tx(struct xhci_hcd *xhci, gfp_t mem_flags, in xhci_queue_bulk_tx() argument
3688 ring = xhci_urb_to_transfer_ring(xhci, urb); in xhci_queue_bulk_tx()
3705 ret = prepare_transfer(xhci, xhci->devs[slot_id], in xhci_queue_bulk_tx()
3754 if (xhci_align_td(xhci, urb, enqd_len, in xhci_queue_bulk_tx()
3782 remainder = xhci_td_remainder(xhci, enqd_len, trb_buff_len, in xhci_queue_bulk_tx()
3789 queue_trb(xhci, ring, more_trbs_coming | need_zero_pkt, in xhci_queue_bulk_tx()
3814 ret = prepare_transfer(xhci, xhci->devs[slot_id], in xhci_queue_bulk_tx()
3820 queue_trb(xhci, ring, 0, 0, 0, TRB_INTR_TARGET(0), field); in xhci_queue_bulk_tx()
3825 giveback_first_trb(xhci, slot_id, ep_index, urb->stream_id, in xhci_queue_bulk_tx()
3831 int xhci_queue_ctrl_tx(struct xhci_hcd *xhci, gfp_t mem_flags, in xhci_queue_ctrl_tx() argument
3844 ep_ring = xhci_urb_to_transfer_ring(xhci, urb); in xhci_queue_ctrl_tx()
3864 ret = prepare_transfer(xhci, xhci->devs[slot_id], in xhci_queue_ctrl_tx()
3891 if ((xhci->hci_version >= 0x100) || (xhci->quirks & XHCI_MTK_HOST)) { in xhci_queue_ctrl_tx()
3900 queue_trb(xhci, ep_ring, true, in xhci_queue_ctrl_tx()
3927 remainder = xhci_td_remainder(xhci, 0, in xhci_queue_ctrl_tx()
3936 queue_trb(xhci, ep_ring, true, in xhci_queue_ctrl_tx()
3953 queue_trb(xhci, ep_ring, false, in xhci_queue_ctrl_tx()
3960 giveback_first_trb(xhci, slot_id, ep_index, 0, in xhci_queue_ctrl_tx()
3973 static unsigned int xhci_get_burst_count(struct xhci_hcd *xhci, in xhci_get_burst_count() argument
3978 if (xhci->hci_version < 0x100 || urb->dev->speed < USB_SPEED_SUPER) in xhci_get_burst_count()
3993 static unsigned int xhci_get_last_burst_packet_count(struct xhci_hcd *xhci, in xhci_get_last_burst_packet_count() argument
3999 if (xhci->hci_version < 0x100) in xhci_get_last_burst_packet_count()
4025 static int xhci_get_isoc_frame_id(struct xhci_hcd *xhci, in xhci_get_isoc_frame_id() argument
4045 ist = HCS_IST(xhci->hcs_params2) & 0x7; in xhci_get_isoc_frame_id()
4046 if (HCS_IST(xhci->hcs_params2) & (1 << 3)) in xhci_get_isoc_frame_id()
4062 current_frame_id = readl(&xhci->run_regs->microframe_index); in xhci_get_isoc_frame_id()
4070 xhci_dbg(xhci, "%s: index %d, reg 0x%x start_frame_id 0x%x, end_frame_id 0x%x, start_frame 0x%x\n", in xhci_get_isoc_frame_id()
4071 __func__, index, readl(&xhci->run_regs->microframe_index), in xhci_get_isoc_frame_id()
4099 xhci_warn(xhci, "Frame ID %d (reg %d, index %d) beyond range (%d, %d)\n", in xhci_get_isoc_frame_id()
4102 xhci_warn(xhci, "Ignore frame ID field, use SIA bit instead\n"); in xhci_get_isoc_frame_id()
4110 static bool trb_block_event_intr(struct xhci_hcd *xhci, int num_tds, int i) in trb_block_event_intr() argument
4112 if (xhci->hci_version < 0x100) in trb_block_event_intr()
4121 if (i && xhci->quirks & XHCI_AVOID_BEI) in trb_block_event_intr()
4128 static int xhci_queue_isoc_tx(struct xhci_hcd *xhci, gfp_t mem_flags, in xhci_queue_isoc_tx() argument
4146 xep = &xhci->devs[slot_id]->eps[ep_index]; in xhci_queue_isoc_tx()
4147 ep_ring = xhci->devs[slot_id]->eps[ep_index].ring; in xhci_queue_isoc_tx()
4151 xhci_dbg(xhci, "Isoc URB with zero packets?\n"); in xhci_queue_isoc_tx()
4176 burst_count = xhci_get_burst_count(xhci, urb, total_pkt_count); in xhci_queue_isoc_tx()
4177 last_burst_pkt_count = xhci_get_last_burst_packet_count(xhci, in xhci_queue_isoc_tx()
4182 ret = prepare_transfer(xhci, xhci->devs[slot_id], ep_index, in xhci_queue_isoc_tx()
4194 HCC_CFC(xhci->hcc_params)) { in xhci_queue_isoc_tx()
4195 frame_id = xhci_get_isoc_frame_id(xhci, urb, i); in xhci_queue_isoc_tx()
4235 if (trb_block_event_intr(xhci, num_tds, i)) in xhci_queue_isoc_tx()
4244 remainder = xhci_td_remainder(xhci, running_total, in xhci_queue_isoc_tx()
4258 queue_trb(xhci, ep_ring, more_trbs_coming, in xhci_queue_isoc_tx()
4271 xhci_err(xhci, "ISOC TD length unmatch\n"); in xhci_queue_isoc_tx()
4278 if (HCC_CFC(xhci->hcc_params)) in xhci_queue_isoc_tx()
4281 if (xhci_to_hcd(xhci)->self.bandwidth_isoc_reqs == 0) { in xhci_queue_isoc_tx()
4282 if (xhci->quirks & XHCI_AMD_PLL_FIX) in xhci_queue_isoc_tx()
4285 xhci_to_hcd(xhci)->self.bandwidth_isoc_reqs++; in xhci_queue_isoc_tx()
4287 giveback_first_trb(xhci, slot_id, ep_index, urb->stream_id, in xhci_queue_isoc_tx()
4303 td_to_noop(xhci, ep_ring, &urb_priv->td[0], true); in xhci_queue_isoc_tx()
4321 int xhci_queue_isoc_tx_prepare(struct xhci_hcd *xhci, gfp_t mem_flags, in xhci_queue_isoc_tx_prepare() argument
4333 xdev = xhci->devs[slot_id]; in xhci_queue_isoc_tx_prepare()
4334 xep = &xhci->devs[slot_id]->eps[ep_index]; in xhci_queue_isoc_tx_prepare()
4336 ep_ctx = xhci_get_ep_ctx(xhci, xdev->out_ctx, ep_index); in xhci_queue_isoc_tx_prepare()
4346 ret = prepare_ring(xhci, ep_ring, GET_EP_CTX_STATE(ep_ctx), in xhci_queue_isoc_tx_prepare()
4355 check_interval(xhci, urb, ep_ctx); in xhci_queue_isoc_tx_prepare()
4358 if (HCC_CFC(xhci->hcc_params) && !list_empty(&ep_ring->td_list)) { in xhci_queue_isoc_tx_prepare()
4365 start_frame = readl(&xhci->run_regs->microframe_index); in xhci_queue_isoc_tx_prepare()
4371 ist = HCS_IST(xhci->hcs_params2) & 0x7; in xhci_queue_isoc_tx_prepare()
4372 if (HCS_IST(xhci->hcs_params2) & (1 << 3)) in xhci_queue_isoc_tx_prepare()
4393 return xhci_queue_isoc_tx(xhci, mem_flags, urb, slot_id, ep_index); in xhci_queue_isoc_tx_prepare()
4406 static int queue_command(struct xhci_hcd *xhci, struct xhci_command *cmd, in queue_command() argument
4410 int reserved_trbs = xhci->cmd_ring_reserved_trbs; in queue_command()
4413 if ((xhci->xhc_state & XHCI_STATE_DYING) || in queue_command()
4414 (xhci->xhc_state & XHCI_STATE_HALTED)) { in queue_command()
4415 xhci_dbg(xhci, "xHCI dying or halted, can't queue_command\n"); in queue_command()
4422 ret = prepare_ring(xhci, xhci->cmd_ring, EP_STATE_RUNNING, in queue_command()
4425 xhci_err(xhci, "ERR: No room for command on command ring\n"); in queue_command()
4427 xhci_err(xhci, "ERR: Reserved TRB counting for " in queue_command()
4432 cmd->command_trb = xhci->cmd_ring->enqueue; in queue_command()
4435 if (list_empty(&xhci->cmd_list)) { in queue_command()
4436 xhci->current_cmd = cmd; in queue_command()
4437 xhci_mod_cmd_timer(xhci, XHCI_CMD_DEFAULT_TIMEOUT); in queue_command()
4440 list_add_tail(&cmd->cmd_list, &xhci->cmd_list); in queue_command()
4442 queue_trb(xhci, xhci->cmd_ring, false, field1, field2, field3, in queue_command()
4443 field4 | xhci->cmd_ring->cycle_state); in queue_command()
4448 int xhci_queue_slot_control(struct xhci_hcd *xhci, struct xhci_command *cmd, in xhci_queue_slot_control() argument
4451 return queue_command(xhci, cmd, 0, 0, 0, in xhci_queue_slot_control()
4456 int xhci_queue_address_device(struct xhci_hcd *xhci, struct xhci_command *cmd, in xhci_queue_address_device() argument
4459 return queue_command(xhci, cmd, lower_32_bits(in_ctx_ptr), in xhci_queue_address_device()
4465 int xhci_queue_vendor_command(struct xhci_hcd *xhci, struct xhci_command *cmd, in xhci_queue_vendor_command() argument
4468 return queue_command(xhci, cmd, field1, field2, field3, field4, false); in xhci_queue_vendor_command()
4472 int xhci_queue_reset_device(struct xhci_hcd *xhci, struct xhci_command *cmd, in xhci_queue_reset_device() argument
4475 return queue_command(xhci, cmd, 0, 0, 0, in xhci_queue_reset_device()
4481 int xhci_queue_configure_endpoint(struct xhci_hcd *xhci, in xhci_queue_configure_endpoint() argument
4485 return queue_command(xhci, cmd, lower_32_bits(in_ctx_ptr), in xhci_queue_configure_endpoint()
4492 int xhci_queue_evaluate_context(struct xhci_hcd *xhci, struct xhci_command *cmd, in xhci_queue_evaluate_context() argument
4495 return queue_command(xhci, cmd, lower_32_bits(in_ctx_ptr), in xhci_queue_evaluate_context()
4505 int xhci_queue_stop_endpoint(struct xhci_hcd *xhci, struct xhci_command *cmd, in xhci_queue_stop_endpoint() argument
4513 return queue_command(xhci, cmd, 0, 0, 0, in xhci_queue_stop_endpoint()
4518 int xhci_queue_reset_ep(struct xhci_hcd *xhci, struct xhci_command *cmd, in xhci_queue_reset_ep() argument
4529 return queue_command(xhci, cmd, 0, 0, 0, in xhci_queue_reset_ep()