Lines Matching +full:additional +full:- +full:devs
1 // SPDX-License-Identifier: GPL-2.0
26 * until you reach a non-link TRB.
57 #include <linux/dma-mapping.h>
59 #include "xhci-trace.h"
60 #include "xhci-mtk.h"
71 if (!seg || !trb || trb < seg->trbs) in xhci_trb_virt_to_dma()
74 segment_offset = trb - seg->trbs; in xhci_trb_virt_to_dma()
77 return seg->dma + (segment_offset * sizeof(*trb)); in xhci_trb_virt_to_dma()
82 return TRB_TYPE_NOOP_LE32(trb->generic.field[3]); in trb_is_noop()
87 return TRB_TYPE_LINK_LE32(trb->link.control); in trb_is_link()
92 return trb == &seg->trbs[TRBS_PER_SEGMENT - 1]; in last_trb_on_seg()
98 return last_trb_on_seg(seg, trb) && (seg->next == ring->first_seg); in last_trb_on_ring()
103 return le32_to_cpu(trb->link.control) & LINK_TOGGLE; in link_trb_toggles_cycle()
108 struct urb_priv *urb_priv = td->urb->hcpriv; in last_td_in_urb()
110 return urb_priv->num_tds_done == urb_priv->num_tds; in last_td_in_urb()
115 struct urb_priv *urb_priv = urb->hcpriv; in inc_td_cnt()
117 urb_priv->num_tds_done++; in inc_td_cnt()
124 trb->link.control &= cpu_to_le32(~TRB_CHAIN); in trb_to_noop()
126 trb->generic.field[0] = 0; in trb_to_noop()
127 trb->generic.field[1] = 0; in trb_to_noop()
128 trb->generic.field[2] = 0; in trb_to_noop()
130 trb->generic.field[3] &= cpu_to_le32(TRB_CYCLE); in trb_to_noop()
131 trb->generic.field[3] |= cpu_to_le32(TRB_TYPE(noop_type)); in trb_to_noop()
145 *seg = (*seg)->next; in next_trb()
146 *trb = ((*seg)->trbs); in next_trb()
159 if (ring->type == TYPE_EVENT) { in inc_deq()
160 if (!last_trb_on_seg(ring->deq_seg, ring->dequeue)) { in inc_deq()
161 ring->dequeue++; in inc_deq()
164 if (last_trb_on_ring(ring, ring->deq_seg, ring->dequeue)) in inc_deq()
165 ring->cycle_state ^= 1; in inc_deq()
166 ring->deq_seg = ring->deq_seg->next; in inc_deq()
167 ring->dequeue = ring->deq_seg->trbs; in inc_deq()
172 if (!trb_is_link(ring->dequeue)) { in inc_deq()
173 ring->dequeue++; in inc_deq()
174 ring->num_trbs_free++; in inc_deq()
176 while (trb_is_link(ring->dequeue)) { in inc_deq()
177 ring->deq_seg = ring->deq_seg->next; in inc_deq()
178 ring->dequeue = ring->deq_seg->trbs; in inc_deq()
210 chain = le32_to_cpu(ring->enqueue->generic.field[3]) & TRB_CHAIN; in inc_enq()
212 if (!trb_is_link(ring->enqueue)) in inc_enq()
213 ring->num_trbs_free--; in inc_enq()
214 next = ++(ring->enqueue); in inc_enq()
233 if (!(ring->type == TYPE_ISOC && in inc_enq()
234 (xhci->quirks & XHCI_AMD_0x96_HOST)) && in inc_enq()
236 next->link.control &= cpu_to_le32(~TRB_CHAIN); in inc_enq()
237 next->link.control |= cpu_to_le32(chain); in inc_enq()
241 next->link.control ^= cpu_to_le32(TRB_CYCLE); in inc_enq()
245 ring->cycle_state ^= 1; in inc_enq()
247 ring->enq_seg = ring->enq_seg->next; in inc_enq()
248 ring->enqueue = ring->enq_seg->trbs; in inc_enq()
249 next = ring->enqueue; in inc_enq()
264 if (ring->num_trbs_free < num_trbs) in room_on_ring()
267 if (ring->type != TYPE_COMMAND && ring->type != TYPE_EVENT) { in room_on_ring()
268 num_trbs_in_deq_seg = ring->dequeue - ring->deq_seg->trbs; in room_on_ring()
269 if (ring->num_trbs_free < num_trbs + num_trbs_in_deq_seg) in room_on_ring()
279 if (!(xhci->cmd_ring_state & CMD_RING_STATE_RUNNING)) in xhci_ring_cmd_db()
286 writel(DB_VALUE_HOST, &xhci->dba->doorbell[0]); in xhci_ring_cmd_db()
288 readl(&xhci->dba->doorbell[0]); in xhci_ring_cmd_db()
293 return mod_delayed_work(system_wq, &xhci->cmd_timer, delay); in xhci_mod_cmd_timer()
298 return list_first_entry_or_null(&xhci->cmd_list, struct xhci_command, in xhci_next_queued_cmd()
303 * Turn all commands on command ring with status set to "aborted" to no-op trbs.
305 * This must be called with command ring stopped and xhci->lock held.
312 /* Turn all aborted commands in list to no-ops, then restart */ in xhci_handle_stopped_cmd_ring()
313 list_for_each_entry(i_cmd, &xhci->cmd_list, cmd_list) { in xhci_handle_stopped_cmd_ring()
315 if (i_cmd->status != COMP_COMMAND_ABORTED) in xhci_handle_stopped_cmd_ring()
318 i_cmd->status = COMP_COMMAND_RING_STOPPED; in xhci_handle_stopped_cmd_ring()
320 xhci_dbg(xhci, "Turn aborted command %p to no-op\n", in xhci_handle_stopped_cmd_ring()
321 i_cmd->command_trb); in xhci_handle_stopped_cmd_ring()
323 trb_to_noop(i_cmd->command_trb, TRB_CMD_NOOP); in xhci_handle_stopped_cmd_ring()
327 * completion event is received for these no-op commands in xhci_handle_stopped_cmd_ring()
331 xhci->cmd_ring_state = CMD_RING_STATE_RUNNING; in xhci_handle_stopped_cmd_ring()
334 if ((xhci->cmd_ring->dequeue != xhci->cmd_ring->enqueue) && in xhci_handle_stopped_cmd_ring()
335 !(xhci->xhc_state & XHCI_STATE_DYING)) { in xhci_handle_stopped_cmd_ring()
336 xhci->current_cmd = cur_cmd; in xhci_handle_stopped_cmd_ring()
343 /* Must be called with xhci->lock held, releases and aquires lock back */
346 struct xhci_segment *new_seg = xhci->cmd_ring->deq_seg; in xhci_abort_cmd_ring()
347 union xhci_trb *new_deq = xhci->cmd_ring->dequeue; in xhci_abort_cmd_ring()
353 reinit_completion(&xhci->cmd_ring_stop_completion); in xhci_abort_cmd_ring()
368 xhci_write_64(xhci, crcr | CMD_RING_ABORT, &xhci->op_regs->cmd_ring); in xhci_abort_cmd_ring()
372 * seconds then driver handles it as if host died (-ENODEV). in xhci_abort_cmd_ring()
373 * In the future we should distinguish between -ENODEV and -ETIMEDOUT in xhci_abort_cmd_ring()
374 * and try to recover a -ETIMEDOUT with a host controller reset. in xhci_abort_cmd_ring()
376 ret = xhci_handshake(&xhci->op_regs->cmd_ring, in xhci_abort_cmd_ring()
390 spin_unlock_irqrestore(&xhci->lock, flags); in xhci_abort_cmd_ring()
391 ret = wait_for_completion_timeout(&xhci->cmd_ring_stop_completion, in xhci_abort_cmd_ring()
393 spin_lock_irqsave(&xhci->lock, flags); in xhci_abort_cmd_ring()
408 __le32 __iomem *db_addr = &xhci->dba->doorbell[slot_id]; in xhci_ring_ep_doorbell()
409 struct xhci_virt_ep *ep = &xhci->devs[slot_id]->eps[ep_index]; in xhci_ring_ep_doorbell()
410 unsigned int ep_state = ep->ep_state; in xhci_ring_ep_doorbell()
426 * write-posting flush. It'll get there soon enough. in xhci_ring_ep_doorbell()
438 ep = &xhci->devs[slot_id]->eps[ep_index]; in ring_doorbell_for_active_rings()
441 if (!(ep->ep_state & EP_HAS_STREAMS)) { in ring_doorbell_for_active_rings()
442 if (ep->ring && !(list_empty(&ep->ring->td_list))) in ring_doorbell_for_active_rings()
447 for (stream_id = 1; stream_id < ep->stream_info->num_streams; in ring_doorbell_for_active_rings()
449 struct xhci_stream_info *stream_info = ep->stream_info; in ring_doorbell_for_active_rings()
450 if (!list_empty(&stream_info->stream_rings[stream_id]->td_list)) in ring_doorbell_for_active_rings()
475 if (!xhci->devs[slot_id]) { in xhci_get_virt_ep()
480 return &xhci->devs[slot_id]->eps[ep_index]; in xhci_get_virt_ep()
498 if (!(ep->ep_state & EP_HAS_STREAMS)) in xhci_triad_to_transfer_ring()
499 return ep->ring; in xhci_triad_to_transfer_ring()
509 if (stream_id < ep->stream_info->num_streams) in xhci_triad_to_transfer_ring()
510 return ep->stream_info->stream_rings[stream_id]; in xhci_triad_to_transfer_ring()
517 ep->stream_info->num_streams - 1, in xhci_triad_to_transfer_ring()
536 ep = &vdev->eps[ep_index]; in xhci_get_hw_deq()
538 if (ep->ep_state & EP_HAS_STREAMS) { in xhci_get_hw_deq()
539 st_ctx = &ep->stream_info->stream_ctx_array[stream_id]; in xhci_get_hw_deq()
540 return le64_to_cpu(st_ctx->stream_ring); in xhci_get_hw_deq()
542 ep_ctx = xhci_get_ep_ctx(xhci, vdev->out_ctx, ep_index); in xhci_get_hw_deq()
543 return le64_to_cpu(ep_ctx->deq); in xhci_get_hw_deq()
553 * - First we update our new ring state to be the same as when the xHC stopped.
554 * - Then we traverse the ring to find the segment that contains
557 * - Finally we move the dequeue state one TRB further, toggling the cycle bit
569 struct xhci_virt_device *dev = xhci->devs[slot_id]; in xhci_find_new_dequeue_state()
570 struct xhci_virt_ep *ep = &dev->eps[ep_index]; in xhci_find_new_dequeue_state()
596 if (list_empty(&ep_ring->td_list)) { in xhci_find_new_dequeue_state()
597 state->new_deq_seg = ep_ring->enq_seg; in xhci_find_new_dequeue_state()
598 state->new_deq_ptr = ep_ring->enqueue; in xhci_find_new_dequeue_state()
599 state->new_cycle_state = ep_ring->cycle_state; in xhci_find_new_dequeue_state()
612 new_seg = ep_ring->deq_seg; in xhci_find_new_dequeue_state()
613 new_deq = ep_ring->dequeue; in xhci_find_new_dequeue_state()
616 * Quirk: xHC write-back of the DCS field in the hardware dequeue in xhci_find_new_dequeue_state()
617 * pointer is wrong - use the cycle state of the TRB pointed to by in xhci_find_new_dequeue_state()
620 if (xhci->quirks & XHCI_EP_CTX_BROKEN_DCS && in xhci_find_new_dequeue_state()
621 !(ep->ep_state & EP_HAS_STREAMS)) in xhci_find_new_dequeue_state()
622 halted_seg = trb_in_td(xhci, cur_td->start_seg, in xhci_find_new_dequeue_state()
623 cur_td->first_trb, cur_td->last_trb, in xhci_find_new_dequeue_state()
626 index = ((dma_addr_t)(hw_dequeue & ~0xf) - halted_seg->dma) / in xhci_find_new_dequeue_state()
628 halted_trb = &halted_seg->trbs[index]; in xhci_find_new_dequeue_state()
629 state->new_cycle_state = halted_trb->generic.field[3] & 0x1; in xhci_find_new_dequeue_state()
632 state->new_cycle_state); in xhci_find_new_dequeue_state()
634 state->new_cycle_state = hw_dequeue & 0x1; in xhci_find_new_dequeue_state()
636 state->stream_id = stream_id; in xhci_find_new_dequeue_state()
651 if (new_deq == cur_td->last_trb) in xhci_find_new_dequeue_state()
656 state->new_cycle_state ^= 0x1; in xhci_find_new_dequeue_state()
661 if (new_deq == ep->ring->dequeue) { in xhci_find_new_dequeue_state()
663 state->new_deq_seg = NULL; in xhci_find_new_dequeue_state()
664 state->new_deq_ptr = NULL; in xhci_find_new_dequeue_state()
670 state->new_deq_seg = new_seg; in xhci_find_new_dequeue_state()
671 state->new_deq_ptr = new_deq; in xhci_find_new_dequeue_state()
676 "Cycle state = 0x%x", state->new_cycle_state); in xhci_find_new_dequeue_state()
680 state->new_deq_seg); in xhci_find_new_dequeue_state()
681 addr = xhci_trb_virt_to_dma(state->new_deq_seg, state->new_deq_ptr); in xhci_find_new_dequeue_state()
694 struct xhci_segment *seg = td->start_seg; in td_to_noop()
695 union xhci_trb *trb = td->first_trb; in td_to_noop()
701 if (flip_cycle && trb != td->first_trb && trb != td->last_trb) in td_to_noop()
702 trb->generic.field[3] ^= cpu_to_le32(TRB_CYCLE); in td_to_noop()
704 if (trb == td->last_trb) in td_to_noop()
714 ep->ep_state &= ~EP_STOP_CMD_PENDING; in xhci_stop_watchdog_timer_in_irq()
716 del_timer(&ep->stop_cmd_timer); in xhci_stop_watchdog_timer_in_irq()
720 * Must be called with xhci->lock held in interrupt context,
721 * releases and re-acquires xhci->lock
726 struct urb *urb = cur_td->urb; in xhci_giveback_urb_in_irq()
727 struct urb_priv *urb_priv = urb->hcpriv; in xhci_giveback_urb_in_irq()
728 struct usb_hcd *hcd = bus_to_hcd(urb->dev->bus); in xhci_giveback_urb_in_irq()
730 if (usb_pipetype(urb->pipe) == PIPE_ISOCHRONOUS) { in xhci_giveback_urb_in_irq()
731 xhci_to_hcd(xhci)->self.bandwidth_isoc_reqs--; in xhci_giveback_urb_in_irq()
732 if (xhci_to_hcd(xhci)->self.bandwidth_isoc_reqs == 0) { in xhci_giveback_urb_in_irq()
733 if (xhci->quirks & XHCI_AMD_PLL_FIX) in xhci_giveback_urb_in_irq()
746 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; in xhci_unmap_td_bounce_buffer()
747 struct xhci_segment *seg = td->bounce_seg; in xhci_unmap_td_bounce_buffer()
748 struct urb *urb = td->urb; in xhci_unmap_td_bounce_buffer()
755 dma_unmap_single(dev, seg->bounce_dma, ring->bounce_buf_len, in xhci_unmap_td_bounce_buffer()
760 dma_unmap_single(dev, seg->bounce_dma, ring->bounce_buf_len, in xhci_unmap_td_bounce_buffer()
763 if (urb->num_sgs) { in xhci_unmap_td_bounce_buffer()
764 len = sg_pcopy_from_buffer(urb->sg, urb->num_sgs, seg->bounce_buf, in xhci_unmap_td_bounce_buffer()
765 seg->bounce_len, seg->bounce_offs); in xhci_unmap_td_bounce_buffer()
766 if (len != seg->bounce_len) in xhci_unmap_td_bounce_buffer()
768 len, seg->bounce_len); in xhci_unmap_td_bounce_buffer()
770 memcpy(urb->transfer_buffer + seg->bounce_offs, seg->bounce_buf, in xhci_unmap_td_bounce_buffer()
771 seg->bounce_len); in xhci_unmap_td_bounce_buffer()
773 seg->bounce_len = 0; in xhci_unmap_td_bounce_buffer()
774 seg->bounce_offs = 0; in xhci_unmap_td_bounce_buffer()
783 urb = td->urb; in xhci_td_cleanup()
790 * length, urb->actual_length will be a very big number (since it's in xhci_td_cleanup()
793 if (urb->actual_length > urb->transfer_buffer_length) { in xhci_td_cleanup()
795 urb->transfer_buffer_length, urb->actual_length); in xhci_td_cleanup()
796 urb->actual_length = 0; in xhci_td_cleanup()
799 list_del_init(&td->td_list); in xhci_td_cleanup()
801 if (!list_empty(&td->cancelled_td_list)) in xhci_td_cleanup()
802 list_del_init(&td->cancelled_td_list); in xhci_td_cleanup()
807 if ((urb->actual_length != urb->transfer_buffer_length && in xhci_td_cleanup()
808 (urb->transfer_flags & URB_SHORT_NOT_OK)) || in xhci_td_cleanup()
809 (status != 0 && !usb_endpoint_xfer_isoc(&urb->ep->desc))) in xhci_td_cleanup()
811 urb, urb->actual_length, in xhci_td_cleanup()
812 urb->transfer_buffer_length, status); in xhci_td_cleanup()
815 if (usb_pipetype(urb->pipe) == PIPE_ISOCHRONOUS) in xhci_td_cleanup()
831 ret = -ENOMEM; in xhci_reset_halted_ep()
848 unsigned int slot_id = ep->vdev->slot_id; in xhci_handle_halted_endpoint()
855 if (ep->vdev->flags & VDEV_PORT_ERROR) in xhci_handle_halted_endpoint()
858 ep->ep_state |= EP_HALTED; in xhci_handle_halted_endpoint()
860 err = xhci_reset_halted_ep(xhci, slot_id, ep->ep_index, reset_type); in xhci_handle_halted_endpoint()
865 ep->ep_state |= EP_HARD_CLEAR_TOGGLE; in xhci_handle_halted_endpoint()
866 xhci_cleanup_stalled_ring(xhci, slot_id, ep->ep_index, stream_id, in xhci_handle_halted_endpoint()
875 * We're also in the event handler, so we can't get re-interrupted if another
888 xhci = ep->xhci; in xhci_invalidate_cancelled_tds()
890 list_for_each_entry_safe(td, tmp_td, &ep->cancelled_td_list, cancelled_td_list) { in xhci_invalidate_cancelled_tds()
894 td->start_seg, td->first_trb)); in xhci_invalidate_cancelled_tds()
895 list_del_init(&td->td_list); in xhci_invalidate_cancelled_tds()
896 ring = xhci_urb_to_transfer_ring(xhci, td->urb); in xhci_invalidate_cancelled_tds()
899 td->urb, td->urb->stream_id); in xhci_invalidate_cancelled_tds()
906 hw_deq = xhci_get_hw_deq(xhci, ep->vdev, ep->ep_index, in xhci_invalidate_cancelled_tds()
907 td->urb->stream_id); in xhci_invalidate_cancelled_tds()
910 if (trb_in_td(xhci, td->start_seg, td->first_trb, in xhci_invalidate_cancelled_tds()
911 td->last_trb, hw_deq, false)) { in xhci_invalidate_cancelled_tds()
912 xhci_find_new_dequeue_state(xhci, ep->vdev->slot_id, in xhci_invalidate_cancelled_tds()
913 ep->ep_index, in xhci_invalidate_cancelled_tds()
914 td->urb->stream_id, in xhci_invalidate_cancelled_tds()
931 * 2. Otherwise, we turn all the TRBs in the TD into No-op TRBs (with the chain
946 if (unlikely(TRB_TO_SUSPEND_PORT(le32_to_cpu(trb->generic.field[3])))) { in xhci_handle_cmd_stop_ep()
947 if (!xhci->devs[slot_id]) in xhci_handle_cmd_stop_ep()
955 ep_index = TRB_TO_EP_INDEX(le32_to_cpu(trb->generic.field[3])); in xhci_handle_cmd_stop_ep()
961 vdev = xhci->devs[slot_id]; in xhci_handle_cmd_stop_ep()
962 ep_ctx = xhci_get_ep_ctx(xhci, vdev->out_ctx, ep_index); in xhci_handle_cmd_stop_ep()
965 last_unlinked_td = list_last_entry(&ep->cancelled_td_list, in xhci_handle_cmd_stop_ep()
968 if (list_empty(&ep->cancelled_td_list)) { in xhci_handle_cmd_stop_ep()
995 cur_td = list_first_entry(&ep->cancelled_td_list, in xhci_handle_cmd_stop_ep()
997 list_del_init(&cur_td->cancelled_td_list); in xhci_handle_cmd_stop_ep()
1003 ep_ring = xhci_urb_to_transfer_ring(xhci, cur_td->urb); in xhci_handle_cmd_stop_ep()
1005 inc_td_cnt(cur_td->urb); in xhci_handle_cmd_stop_ep()
1012 if (xhci->xhc_state & XHCI_STATE_DYING) in xhci_handle_cmd_stop_ep()
1016 /* Return to the event handler with xhci->lock re-acquired */ in xhci_handle_cmd_stop_ep()
1024 list_for_each_entry_safe(cur_td, tmp, &ring->td_list, td_list) { in xhci_kill_ring_urbs()
1025 list_del_init(&cur_td->td_list); in xhci_kill_ring_urbs()
1027 if (!list_empty(&cur_td->cancelled_td_list)) in xhci_kill_ring_urbs()
1028 list_del_init(&cur_td->cancelled_td_list); in xhci_kill_ring_urbs()
1032 inc_td_cnt(cur_td->urb); in xhci_kill_ring_urbs()
1034 xhci_giveback_urb_in_irq(xhci, cur_td, -ESHUTDOWN); in xhci_kill_ring_urbs()
1050 if ((ep->ep_state & EP_HAS_STREAMS) || in xhci_kill_endpoint_urbs()
1051 (ep->ep_state & EP_GETTING_NO_STREAMS)) { in xhci_kill_endpoint_urbs()
1054 for (stream_id = 1; stream_id < ep->stream_info->num_streams; in xhci_kill_endpoint_urbs()
1056 ring = ep->stream_info->stream_rings[stream_id]; in xhci_kill_endpoint_urbs()
1066 ring = ep->ring; in xhci_kill_endpoint_urbs()
1075 list_for_each_entry_safe(cur_td, tmp, &ep->cancelled_td_list, in xhci_kill_endpoint_urbs()
1077 list_del_init(&cur_td->cancelled_td_list); in xhci_kill_endpoint_urbs()
1078 inc_td_cnt(cur_td->urb); in xhci_kill_endpoint_urbs()
1081 xhci_giveback_urb_in_irq(xhci, cur_td, -ESHUTDOWN); in xhci_kill_endpoint_urbs()
1091 * Call with xhci->lock held.
1092 * lock is relased and re-acquired while giving back urb.
1098 if (xhci->xhc_state & XHCI_STATE_DYING) in xhci_hc_died()
1102 xhci->xhc_state |= XHCI_STATE_DYING; in xhci_hc_died()
1107 for (i = 0; i <= HCS_MAX_SLOTS(xhci->hcs_params1); i++) { in xhci_hc_died()
1108 if (!xhci->devs[i]) in xhci_hc_died()
1115 if (!(xhci->xhc_state & XHCI_STATE_REMOVING)) in xhci_hc_died()
1123 * through xhci->state.
1139 struct xhci_hcd *xhci = ep->xhci; in xhci_stop_endpoint_command_watchdog()
1144 spin_lock_irqsave(&xhci->lock, flags); in xhci_stop_endpoint_command_watchdog()
1147 if (!(ep->ep_state & EP_STOP_CMD_PENDING) || in xhci_stop_endpoint_command_watchdog()
1148 timer_pending(&ep->stop_cmd_timer)) { in xhci_stop_endpoint_command_watchdog()
1149 spin_unlock_irqrestore(&xhci->lock, flags); in xhci_stop_endpoint_command_watchdog()
1153 usbsts = readl(&xhci->op_regs->status); in xhci_stop_endpoint_command_watchdog()
1158 ep->ep_state &= ~EP_STOP_CMD_PENDING; in xhci_stop_endpoint_command_watchdog()
1163 * handle a stop endpoint cmd timeout as if host died (-ENODEV). in xhci_stop_endpoint_command_watchdog()
1164 * In the future we could distinguish between -ENODEV and -ETIMEDOUT in xhci_stop_endpoint_command_watchdog()
1165 * and try to recover a -ETIMEDOUT with a host controller reset in xhci_stop_endpoint_command_watchdog()
1169 spin_unlock_irqrestore(&xhci->lock, flags); in xhci_stop_endpoint_command_watchdog()
1183 num_trbs_free_temp = ep_ring->num_trbs_free; in update_ring_for_set_deq_completion()
1184 dequeue_temp = ep_ring->dequeue; in update_ring_for_set_deq_completion()
1186 /* If we get two back-to-back stalls, and the first stalled transfer in update_ring_for_set_deq_completion()
1190 * the segment into la-la-land. in update_ring_for_set_deq_completion()
1192 if (trb_is_link(ep_ring->dequeue)) { in update_ring_for_set_deq_completion()
1193 ep_ring->deq_seg = ep_ring->deq_seg->next; in update_ring_for_set_deq_completion()
1194 ep_ring->dequeue = ep_ring->deq_seg->trbs; in update_ring_for_set_deq_completion()
1197 while (ep_ring->dequeue != dev->eps[ep_index].queued_deq_ptr) { in update_ring_for_set_deq_completion()
1199 ep_ring->num_trbs_free++; in update_ring_for_set_deq_completion()
1200 ep_ring->dequeue++; in update_ring_for_set_deq_completion()
1201 if (trb_is_link(ep_ring->dequeue)) { in update_ring_for_set_deq_completion()
1202 if (ep_ring->dequeue == in update_ring_for_set_deq_completion()
1203 dev->eps[ep_index].queued_deq_ptr) in update_ring_for_set_deq_completion()
1205 ep_ring->deq_seg = ep_ring->deq_seg->next; in update_ring_for_set_deq_completion()
1206 ep_ring->dequeue = ep_ring->deq_seg->trbs; in update_ring_for_set_deq_completion()
1208 if (ep_ring->dequeue == dequeue_temp) { in update_ring_for_set_deq_completion()
1216 ep_ring->num_trbs_free = num_trbs_free_temp; in update_ring_for_set_deq_completion()
1238 ep_index = TRB_TO_EP_INDEX(le32_to_cpu(trb->generic.field[3])); in xhci_handle_cmd_set_deq()
1239 stream_id = TRB_TO_STREAM_ID(le32_to_cpu(trb->generic.field[2])); in xhci_handle_cmd_set_deq()
1244 dev = xhci->devs[slot_id]; in xhci_handle_cmd_set_deq()
1253 ep_ctx = xhci_get_ep_ctx(xhci, dev->out_ctx, ep_index); in xhci_handle_cmd_set_deq()
1254 slot_ctx = xhci_get_slot_ctx(xhci, dev->out_ctx); in xhci_handle_cmd_set_deq()
1269 slot_state = le32_to_cpu(slot_ctx->dev_state); in xhci_handle_cmd_set_deq()
1293 if (ep->ep_state & EP_HAS_STREAMS) { in xhci_handle_cmd_set_deq()
1295 &ep->stream_info->stream_ctx_array[stream_id]; in xhci_handle_cmd_set_deq()
1296 deq = le64_to_cpu(ctx->stream_ring) & SCTX_DEQ_MASK; in xhci_handle_cmd_set_deq()
1298 deq = le64_to_cpu(ep_ctx->deq) & ~EP_CTX_CYCLE_MASK; in xhci_handle_cmd_set_deq()
1302 if (xhci_trb_virt_to_dma(ep->queued_deq_seg, in xhci_handle_cmd_set_deq()
1303 ep->queued_deq_ptr) == deq) { in xhci_handle_cmd_set_deq()
1312 ep->queued_deq_seg, ep->queued_deq_ptr); in xhci_handle_cmd_set_deq()
1317 ep->ep_state &= ~SET_DEQ_PENDING; in xhci_handle_cmd_set_deq()
1318 ep->queued_deq_seg = NULL; in xhci_handle_cmd_set_deq()
1319 ep->queued_deq_ptr = NULL; in xhci_handle_cmd_set_deq()
1332 ep_index = TRB_TO_EP_INDEX(le32_to_cpu(trb->generic.field[3])); in xhci_handle_cmd_reset_ep()
1337 vdev = xhci->devs[slot_id]; in xhci_handle_cmd_reset_ep()
1338 ep_ctx = xhci_get_ep_ctx(xhci, vdev->out_ctx, ep_index); in xhci_handle_cmd_reset_ep()
1351 if (xhci->quirks & XHCI_RESET_EP_QUIRK) { in xhci_handle_cmd_reset_ep()
1361 xhci->devs[slot_id]->in_ctx->dma, slot_id, in xhci_handle_cmd_reset_ep()
1366 ep->ep_state &= ~EP_HALTED; in xhci_handle_cmd_reset_ep()
1370 if ((le32_to_cpu(trb->generic.field[3])) & TRB_TSP) in xhci_handle_cmd_reset_ep()
1378 command->slot_id = slot_id; in xhci_handle_cmd_enable_slot()
1380 command->slot_id = 0; in xhci_handle_cmd_enable_slot()
1388 virt_dev = xhci->devs[slot_id]; in xhci_handle_cmd_disable_slot()
1392 slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->out_ctx); in xhci_handle_cmd_disable_slot()
1395 if (xhci->quirks & XHCI_EP_LIMIT_QUIRK) in xhci_handle_cmd_disable_slot()
1418 virt_dev = xhci->devs[slot_id]; in xhci_handle_cmd_config_ep()
1419 ctrl_ctx = xhci_get_input_control_ctx(virt_dev->in_ctx); in xhci_handle_cmd_config_ep()
1425 add_flags = le32_to_cpu(ctrl_ctx->add_flags); in xhci_handle_cmd_config_ep()
1426 drop_flags = le32_to_cpu(ctrl_ctx->drop_flags); in xhci_handle_cmd_config_ep()
1428 ep_index = xhci_last_valid_endpoint(add_flags) - 1; in xhci_handle_cmd_config_ep()
1430 ep_ctx = xhci_get_ep_ctx(xhci, virt_dev->out_ctx, ep_index); in xhci_handle_cmd_config_ep()
1439 if (xhci->quirks & XHCI_RESET_EP_QUIRK && in xhci_handle_cmd_config_ep()
1440 ep_index != (unsigned int) -1 && in xhci_handle_cmd_config_ep()
1441 add_flags - SLOT_FLAG == drop_flags) { in xhci_handle_cmd_config_ep()
1442 ep_state = virt_dev->eps[ep_index].ep_state; in xhci_handle_cmd_config_ep()
1446 "Completed config ep cmd - " in xhci_handle_cmd_config_ep()
1450 virt_dev->eps[ep_index].ep_state &= ~EP_HALTED; in xhci_handle_cmd_config_ep()
1462 vdev = xhci->devs[slot_id]; in xhci_handle_cmd_addr_dev()
1463 slot_ctx = xhci_get_slot_ctx(xhci, vdev->out_ctx); in xhci_handle_cmd_addr_dev()
1473 vdev = xhci->devs[slot_id]; in xhci_handle_cmd_reset_dev()
1474 slot_ctx = xhci_get_slot_ctx(xhci, vdev->out_ctx); in xhci_handle_cmd_reset_dev()
1478 if (!xhci->devs[slot_id]) in xhci_handle_cmd_reset_dev()
1486 if (!(xhci->quirks & XHCI_NEC_HOST)) { in xhci_handle_cmd_nec_get_fw()
1487 xhci_warn(xhci, "WARN NEC_GET_FW command on non-NEC host\n"); in xhci_handle_cmd_nec_get_fw()
1492 NEC_FW_MAJOR(le32_to_cpu(event->status)), in xhci_handle_cmd_nec_get_fw()
1493 NEC_FW_MINOR(le32_to_cpu(event->status))); in xhci_handle_cmd_nec_get_fw()
1498 list_del(&cmd->cmd_list); in xhci_complete_del_and_free_cmd()
1500 if (cmd->completion) { in xhci_complete_del_and_free_cmd()
1501 cmd->status = status; in xhci_complete_del_and_free_cmd()
1502 complete(cmd->completion); in xhci_complete_del_and_free_cmd()
1511 xhci->current_cmd = NULL; in xhci_cleanup_command_queue()
1512 list_for_each_entry_safe(cur_cmd, tmp_cmd, &xhci->cmd_list, cmd_list) in xhci_cleanup_command_queue()
1524 spin_lock_irqsave(&xhci->lock, flags); in xhci_handle_command_timeout()
1530 if (!xhci->current_cmd || delayed_work_pending(&xhci->cmd_timer)) { in xhci_handle_command_timeout()
1531 spin_unlock_irqrestore(&xhci->lock, flags); in xhci_handle_command_timeout()
1535 xhci->current_cmd->status = COMP_COMMAND_ABORTED; in xhci_handle_command_timeout()
1538 hw_ring_state = xhci_read_64(xhci, &xhci->op_regs->cmd_ring); in xhci_handle_command_timeout()
1544 if ((xhci->cmd_ring_state & CMD_RING_STATE_RUNNING) && in xhci_handle_command_timeout()
1547 xhci->cmd_ring_state = CMD_RING_STATE_ABORTED; in xhci_handle_command_timeout()
1554 if (xhci->xhc_state & XHCI_STATE_REMOVING) { in xhci_handle_command_timeout()
1563 xhci_handle_stopped_cmd_ring(xhci, xhci->current_cmd); in xhci_handle_command_timeout()
1566 spin_unlock_irqrestore(&xhci->lock, flags); in xhci_handle_command_timeout()
1573 int slot_id = TRB_TO_SLOT_ID(le32_to_cpu(event->flags)); in handle_cmd_completion()
1581 cmd_dma = le64_to_cpu(event->cmd_trb); in handle_cmd_completion()
1582 cmd_trb = xhci->cmd_ring->dequeue; in handle_cmd_completion()
1584 trace_xhci_handle_command(xhci->cmd_ring, &cmd_trb->generic); in handle_cmd_completion()
1586 cmd_dequeue_dma = xhci_trb_virt_to_dma(xhci->cmd_ring->deq_seg, in handle_cmd_completion()
1598 cmd = list_first_entry(&xhci->cmd_list, struct xhci_command, cmd_list); in handle_cmd_completion()
1600 cancel_delayed_work(&xhci->cmd_timer); in handle_cmd_completion()
1602 cmd_comp_code = GET_COMP_CODE(le32_to_cpu(event->status)); in handle_cmd_completion()
1606 complete_all(&xhci->cmd_ring_stop_completion); in handle_cmd_completion()
1610 if (cmd->command_trb != xhci->cmd_ring->dequeue) { in handle_cmd_completion()
1623 xhci->cmd_ring_state = CMD_RING_STATE_STOPPED; in handle_cmd_completion()
1624 if (cmd->status == COMP_COMMAND_ABORTED) { in handle_cmd_completion()
1625 if (xhci->current_cmd == cmd) in handle_cmd_completion()
1626 xhci->current_cmd = NULL; in handle_cmd_completion()
1631 cmd_type = TRB_FIELD_TO_TYPE(le32_to_cpu(cmd_trb->generic.field[3])); in handle_cmd_completion()
1640 if (!cmd->completion) in handle_cmd_completion()
1651 le32_to_cpu(cmd_trb->generic.field[3]))); in handle_cmd_completion()
1652 if (!cmd->completion) in handle_cmd_completion()
1657 le32_to_cpu(cmd_trb->generic.field[3]))); in handle_cmd_completion()
1661 /* Is this an aborted command turned to NO-OP? */ in handle_cmd_completion()
1662 if (cmd->status == COMP_COMMAND_RING_STOPPED) in handle_cmd_completion()
1667 le32_to_cpu(cmd_trb->generic.field[3]))); in handle_cmd_completion()
1675 le32_to_cpu(cmd_trb->generic.field[3])); in handle_cmd_completion()
1688 if (!list_is_singular(&xhci->cmd_list)) { in handle_cmd_completion()
1689 xhci->current_cmd = list_first_entry(&cmd->cmd_list, in handle_cmd_completion()
1692 } else if (xhci->current_cmd == cmd) { in handle_cmd_completion()
1693 xhci->current_cmd = NULL; in handle_cmd_completion()
1699 inc_deq(xhci, xhci->cmd_ring); in handle_cmd_completion()
1707 trb_type = TRB_FIELD_TO_TYPE(le32_to_cpu(event->generic.field[3])); in handle_vendor_event()
1709 if (trb_type == TRB_NEC_CMD_COMP && (xhci->quirks & XHCI_NEC_HOST)) in handle_vendor_event()
1710 handle_cmd_completion(xhci, &event->event_cmd); in handle_vendor_event()
1719 slot_id = TRB_TO_SLOT_ID(le32_to_cpu(event->generic.field[3])); in handle_device_notification()
1720 if (!xhci->devs[slot_id]) { in handle_device_notification()
1728 udev = xhci->devs[slot_id]->udev; in handle_device_notification()
1729 if (udev && udev->parent) in handle_device_notification()
1730 usb_wakeup_notification(udev->parent, udev->portnum); in handle_device_notification()
1736 * As per ThunderX2errata-129 USB 2 device may come up as USB 1
1753 writel(0x6F, hcd->regs + 0x1048); in xhci_cavium_reset_phy_quirk()
1755 /* De-assert the PHY reset */ in xhci_cavium_reset_phy_quirk()
1756 writel(0x7F, hcd->regs + 0x1048); in xhci_cavium_reset_phy_quirk()
1758 pll_lock_check = readl(hcd->regs + 0x1070); in xhci_cavium_reset_phy_quirk()
1759 } while (!(pll_lock_check & 0x1) && --retry_count); in xhci_cavium_reset_phy_quirk()
1776 if (GET_COMP_CODE(le32_to_cpu(event->generic.field[2])) != COMP_SUCCESS) in handle_port_status()
1780 port_id = GET_PORT_ID(le32_to_cpu(event->generic.field[0])); in handle_port_status()
1781 max_ports = HCS_MAX_PORTS(xhci->hcs_params1); in handle_port_status()
1786 inc_deq(xhci, xhci->event_ring); in handle_port_status()
1790 port = &xhci->hw_ports[port_id - 1]; in handle_port_status()
1791 if (!port || !port->rhub || port->hcd_portnum == DUPLICATE_ENTRY) { in handle_port_status()
1799 if (port->rhub == &xhci->usb3_rhub && xhci->shared_hcd == NULL) { in handle_port_status()
1805 hcd = port->rhub->hcd; in handle_port_status()
1806 bus_state = &port->rhub->bus_state; in handle_port_status()
1807 hcd_portnum = port->hcd_portnum; in handle_port_status()
1808 portsc = readl(port->addr); in handle_port_status()
1810 xhci_dbg(xhci, "Port change event, %d-%d, id %d, portsc: 0x%x\n", in handle_port_status()
1811 hcd->self.busnum, hcd_portnum + 1, port_id, portsc); in handle_port_status()
1815 if (hcd->state == HC_STATE_SUSPENDED) { in handle_port_status()
1820 if (hcd->speed >= HCD_USB3 && in handle_port_status()
1823 if (slot_id && xhci->devs[slot_id]) in handle_port_status()
1824 xhci->devs[slot_id]->flags |= VDEV_PORT_ERROR; in handle_port_status()
1830 cmd_reg = readl(&xhci->op_regs->command); in handle_port_status()
1842 bus_state->port_remote_wakeup |= 1 << hcd_portnum; in handle_port_status()
1844 usb_hcd_start_port_resume(&hcd->self, hcd_portnum); in handle_port_status()
1851 } else if (!test_bit(hcd_portnum, &bus_state->resuming_ports)) { in handle_port_status()
1853 port->resume_timestamp = jiffies + in handle_port_status()
1855 set_bit(hcd_portnum, &bus_state->resuming_ports); in handle_port_status()
1858 * usb device auto-resume latency around ~40ms. in handle_port_status()
1860 set_bit(HCD_FLAG_POLL_RH, &hcd->flags); in handle_port_status()
1861 mod_timer(&hcd->rh_timer, in handle_port_status()
1862 port->resume_timestamp); in handle_port_status()
1863 usb_hcd_start_port_resume(&hcd->self, hcd_portnum); in handle_port_status()
1874 complete(&port->u3exit_done); in handle_port_status()
1877 * U3Exit state after a host-initiated resume. If it's a device in handle_port_status()
1883 if (slot_id && xhci->devs[slot_id]) in handle_port_status()
1885 if (bus_state->port_remote_wakeup & (1 << hcd_portnum)) { in handle_port_status()
1887 usb_wakeup_notification(hcd->self.root_hub, in handle_port_status()
1895 * Check to see if xhci-hub.c is waiting on RExit to U0 transition (or in handle_port_status()
1899 if (hcd->speed < HCD_USB3 && port->rexit_active) { in handle_port_status()
1900 complete(&port->rexit_done); in handle_port_status()
1901 port->rexit_active = false; in handle_port_status()
1906 if (hcd->speed < HCD_USB3) { in handle_port_status()
1908 if ((xhci->quirks & XHCI_RESET_PLL_ON_DISCONNECT) && in handle_port_status()
1915 inc_deq(xhci, xhci->event_ring); in handle_port_status()
1925 * xHCI port-status-change events occur when the "or" of all the in handle_port_status()
1926 * status-change bits in the portsc register changes from 0 to 1. in handle_port_status()
1932 set_bit(HCD_FLAG_POLL_RH, &hcd->flags); in handle_port_status()
1933 spin_unlock(&xhci->lock); in handle_port_status()
1936 spin_lock(&xhci->lock); in handle_port_status()
1965 &cur_seg->trbs[TRBS_PER_SEGMENT - 1]); in trb_in_td()
1971 …"Looking for event-dma %016llx trb-start %016llx trb-end %016llx seg-start %016llx seg-end %016llx… in trb_in_td()
1975 (unsigned long long)cur_seg->dma, in trb_in_td()
1989 (suspect_dma >= cur_seg->dma && in trb_in_td()
1999 cur_seg = cur_seg->next; in trb_in_td()
2000 start_dma = xhci_trb_virt_to_dma(cur_seg, &cur_seg->trbs[0]); in trb_in_td()
2010 * As part of low/full-speed endpoint-halt processing in xhci_clear_hub_tt_buffer()
2013 if (td->urb->dev->tt && !usb_pipeint(td->urb->pipe) && in xhci_clear_hub_tt_buffer()
2014 (td->urb->dev->tt->hub != xhci_to_hcd(xhci)->self.root_hub) && in xhci_clear_hub_tt_buffer()
2015 !(ep->ep_state & EP_CLEARING_TT)) { in xhci_clear_hub_tt_buffer()
2016 ep->ep_state |= EP_CLEARING_TT; in xhci_clear_hub_tt_buffer()
2017 td->urb->ep->hcpriv = td->urb->dev; in xhci_clear_hub_tt_buffer()
2018 if (usb_hub_clear_tt_buffer(td->urb)) in xhci_clear_hub_tt_buffer()
2019 ep->ep_state &= ~EP_CLEARING_TT; in xhci_clear_hub_tt_buffer()
2024 * cleanup the halt for a non-default control endpoint if we indicate a stall.
2053 * treat as not-an-error. in xhci_is_vendor_info_code()
2069 ep_ctx = xhci_get_ep_ctx(xhci, ep->vdev->out_ctx, ep->ep_index); in finish_td()
2093 if ((ep->ep_index != 0) || (trb_comp_code != COMP_STALL_ERROR)) in finish_td()
2096 xhci_handle_halted_endpoint(xhci, ep, ep_ring->stream_id, td, in finish_td()
2100 ep_ring->dequeue = td->last_trb; in finish_td()
2101 ep_ring->deq_seg = td->last_trb_seg; in finish_td()
2102 ep_ring->num_trbs_free += td->num_trbs - 1; in finish_td()
2106 return xhci_td_cleanup(xhci, td, ep_ring, td->status); in finish_td()
2114 union xhci_trb *trb = ring->dequeue; in sum_trb_lengths()
2115 struct xhci_segment *seg = ring->deq_seg; in sum_trb_lengths()
2119 sum += TRB_LEN(le32_to_cpu(trb->generic.field[2])); in sum_trb_lengths()
2136 trb_type = TRB_FIELD_TO_TYPE(le32_to_cpu(ep_trb->generic.field[3])); in process_ctrl_td()
2137 ep_ctx = xhci_get_ep_ctx(xhci, ep->vdev->out_ctx, ep->ep_index); in process_ctrl_td()
2138 trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len)); in process_ctrl_td()
2139 requested = td->urb->transfer_buffer_length; in process_ctrl_td()
2140 remaining = EVENT_TRB_LEN(le32_to_cpu(event->transfer_len)); in process_ctrl_td()
2147 td->status = -ESHUTDOWN; in process_ctrl_td()
2150 td->status = 0; in process_ctrl_td()
2153 td->status = 0; in process_ctrl_td()
2157 td->urb->actual_length = remaining; in process_ctrl_td()
2164 td->urb->actual_length = 0; in process_ctrl_td()
2168 td->urb->actual_length = requested - remaining; in process_ctrl_td()
2171 td->urb->actual_length = requested; in process_ctrl_td()
2185 trb_comp_code, ep->ep_index); in process_ctrl_td()
2190 td->urb->actual_length = requested - remaining; in process_ctrl_td()
2191 else if (!td->urb_length_set) in process_ctrl_td()
2192 td->urb->actual_length = 0; in process_ctrl_td()
2206 td->urb_length_set = true; in process_ctrl_td()
2207 td->urb->actual_length = requested - remaining; in process_ctrl_td()
2213 if (!td->urb_length_set) in process_ctrl_td()
2214 td->urb->actual_length = requested; in process_ctrl_td()
2235 trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len)); in process_isoc_td()
2236 urb_priv = td->urb->hcpriv; in process_isoc_td()
2237 idx = urb_priv->num_tds_done; in process_isoc_td()
2238 frame = &td->urb->iso_frame_desc[idx]; in process_isoc_td()
2239 requested = frame->length; in process_isoc_td()
2240 remaining = EVENT_TRB_LEN(le32_to_cpu(event->transfer_len)); in process_isoc_td()
2241 ep_trb_len = TRB_LEN(le32_to_cpu(ep_trb->generic.field[2])); in process_isoc_td()
2242 short_framestatus = td->urb->transfer_flags & URB_SHORT_NOT_OK ? in process_isoc_td()
2243 -EREMOTEIO : 0; in process_isoc_td()
2249 if (td->error_mid_td) in process_isoc_td()
2252 frame->status = short_framestatus; in process_isoc_td()
2253 if (xhci->quirks & XHCI_TRUST_TX_LENGTH) in process_isoc_td()
2257 frame->status = 0; in process_isoc_td()
2260 frame->status = short_framestatus; in process_isoc_td()
2264 frame->status = -ECOMM; in process_isoc_td()
2270 frame->status = -EOVERFLOW; in process_isoc_td()
2271 if (ep_trb != td->last_trb) in process_isoc_td()
2272 td->error_mid_td = true; in process_isoc_td()
2276 frame->status = -EPROTO; in process_isoc_td()
2279 frame->status = -EPROTO; in process_isoc_td()
2281 if (ep_trb != td->last_trb) in process_isoc_td()
2282 td->error_mid_td = true; in process_isoc_td()
2289 frame->status = short_framestatus; in process_isoc_td()
2298 frame->status = -1; in process_isoc_td()
2302 if (td->urb_length_set) in process_isoc_td()
2306 frame->actual_length = sum_trb_lengths(xhci, ep->ring, ep_trb) + in process_isoc_td()
2307 ep_trb_len - remaining; in process_isoc_td()
2309 frame->actual_length = requested; in process_isoc_td()
2311 td->urb->actual_length += frame->actual_length; in process_isoc_td()
2315 if (td->error_mid_td && ep_trb != td->last_trb) { in process_isoc_td()
2317 td->urb_length_set = true; in process_isoc_td()
2331 urb_priv = td->urb->hcpriv; in skip_isoc_td()
2332 idx = urb_priv->num_tds_done; in skip_isoc_td()
2333 frame = &td->urb->iso_frame_desc[idx]; in skip_isoc_td()
2336 frame->status = -EXDEV; in skip_isoc_td()
2339 frame->actual_length = 0; in skip_isoc_td()
2342 ep->ring->dequeue = td->last_trb; in skip_isoc_td()
2343 ep->ring->deq_seg = td->last_trb_seg; in skip_isoc_td()
2344 ep->ring->num_trbs_free += td->num_trbs - 1; in skip_isoc_td()
2345 inc_deq(xhci, ep->ring); in skip_isoc_td()
2347 return xhci_td_cleanup(xhci, td, ep->ring, status); in skip_isoc_td()
2361 slot_ctx = xhci_get_slot_ctx(xhci, ep->vdev->out_ctx); in process_bulk_intr_td()
2362 trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len)); in process_bulk_intr_td()
2363 remaining = EVENT_TRB_LEN(le32_to_cpu(event->transfer_len)); in process_bulk_intr_td()
2364 ep_trb_len = TRB_LEN(le32_to_cpu(ep_trb->generic.field[2])); in process_bulk_intr_td()
2365 requested = td->urb->transfer_buffer_length; in process_bulk_intr_td()
2369 ep->err_count = 0; in process_bulk_intr_td()
2371 if (ep_trb != td->last_trb || remaining) { in process_bulk_intr_td()
2373 xhci_dbg(xhci, "ep %#x - asked for %d bytes, %d bytes untransferred\n", in process_bulk_intr_td()
2374 td->urb->ep->desc.bEndpointAddress, in process_bulk_intr_td()
2377 td->status = 0; in process_bulk_intr_td()
2380 xhci_dbg(xhci, "ep %#x - asked for %d bytes, %d bytes untransferred\n", in process_bulk_intr_td()
2381 td->urb->ep->desc.bEndpointAddress, in process_bulk_intr_td()
2383 td->status = 0; in process_bulk_intr_td()
2386 td->urb->actual_length = remaining; in process_bulk_intr_td()
2394 if (xhci->quirks & XHCI_NO_SOFT_RETRY || in process_bulk_intr_td()
2395 (ep->err_count++ > MAX_SOFT_RETRY) || in process_bulk_intr_td()
2396 le32_to_cpu(slot_ctx->tt_info) & TT_SLOT) in process_bulk_intr_td()
2399 td->status = 0; in process_bulk_intr_td()
2401 xhci_handle_halted_endpoint(xhci, ep, ep_ring->stream_id, td, in process_bulk_intr_td()
2409 if (ep_trb == td->last_trb) in process_bulk_intr_td()
2410 td->urb->actual_length = requested - remaining; in process_bulk_intr_td()
2412 td->urb->actual_length = in process_bulk_intr_td()
2414 ep_trb_len - remaining; in process_bulk_intr_td()
2419 td->urb->actual_length = 0; in process_bulk_intr_td()
2442 int status = -EINPROGRESS; in handle_tx_event()
2449 slot_id = TRB_TO_SLOT_ID(le32_to_cpu(event->flags)); in handle_tx_event()
2450 ep_index = TRB_TO_EP_ID(le32_to_cpu(event->flags)) - 1; in handle_tx_event()
2451 trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len)); in handle_tx_event()
2452 ep_trb_dma = le64_to_cpu(event->buffer); in handle_tx_event()
2460 xdev = xhci->devs[slot_id]; in handle_tx_event()
2462 ep_ctx = xhci_get_ep_ctx(xhci, xdev->out_ctx, ep_index); in handle_tx_event()
2480 if (ep->err_count++ > MAX_SOFT_RETRY) in handle_tx_event()
2499 /* Count current td numbers if ep->skip is set */ in handle_tx_event()
2500 if (ep->skip) { in handle_tx_event()
2501 list_for_each(tmp, &ep_ring->td_list) in handle_tx_event()
2511 if (EVENT_TRB_LEN(le32_to_cpu(event->transfer_len)) == 0) in handle_tx_event()
2513 if (xhci->quirks & XHCI_TRUST_TX_LENGTH || in handle_tx_event()
2514 ep_ring->last_td_was_short) in handle_tx_event()
2529 "Stopped on No-op or Link TRB for slot %u ep %u\n", in handle_tx_event()
2541 ep->ep_state |= EP_HALTED; in handle_tx_event()
2542 status = -EPIPE; in handle_tx_event()
2547 status = -EPROTO; in handle_tx_event()
2552 status = -EPROTO; in handle_tx_event()
2557 status = -EOVERFLOW; in handle_tx_event()
2564 status = -EILSEQ; in handle_tx_event()
2571 status = -ENOSR; in handle_tx_event()
2590 if (!list_empty(&ep_ring->td_list)) in handle_tx_event()
2593 TRB_TO_SLOT_ID(le32_to_cpu(event->flags)), in handle_tx_event()
2598 if (!list_empty(&ep_ring->td_list)) in handle_tx_event()
2601 TRB_TO_SLOT_ID(le32_to_cpu(event->flags)), in handle_tx_event()
2611 ep->skip = true; in handle_tx_event()
2617 ep->skip = true; in handle_tx_event()
2628 status = -EPROTO; in handle_tx_event()
2645 if (list_empty(&ep_ring->td_list)) { in handle_tx_event()
2656 ep_ring->last_td_was_short)) { in handle_tx_event()
2658 TRB_TO_SLOT_ID(le32_to_cpu(event->flags)), in handle_tx_event()
2661 if (ep->skip) { in handle_tx_event()
2662 ep->skip = false; in handle_tx_event()
2670 ep_ring->stream_id, in handle_tx_event()
2677 /* We've skipped all the TDs on the ep ring when ep->skip set */ in handle_tx_event()
2678 if (ep->skip && td_num == 0) { in handle_tx_event()
2679 ep->skip = false; in handle_tx_event()
2685 td = list_first_entry(&ep_ring->td_list, struct xhci_td, in handle_tx_event()
2687 if (ep->skip) in handle_tx_event()
2688 td_num--; in handle_tx_event()
2691 ep_seg = trb_in_td(xhci, ep_ring->deq_seg, ep_ring->dequeue, in handle_tx_event()
2692 td->last_trb, ep_trb_dma, false); in handle_tx_event()
2696 * is not in the current TD pointed by ep_ring->dequeue because in handle_tx_event()
2709 if (ep->skip && usb_endpoint_xfer_isoc(&td->urb->ep->desc)) { in handle_tx_event()
2718 if ((xhci->quirks & XHCI_SPURIOUS_SUCCESS) && in handle_tx_event()
2719 ep_ring->last_td_was_short) { in handle_tx_event()
2720 ep_ring->last_td_was_short = false; in handle_tx_event()
2729 * xhci 4.9.1 states that if there are errors in mult-TRB in handle_tx_event()
2735 if (td->error_mid_td && in handle_tx_event()
2736 !list_is_last(&td->td_list, &ep_ring->td_list)) { in handle_tx_event()
2739 ep_seg = trb_in_td(xhci, td_next->start_seg, td_next->first_trb, in handle_tx_event()
2740 td_next->last_trb, ep_trb_dma, false); in handle_tx_event()
2744 ep_ring->dequeue = td->last_trb; in handle_tx_event()
2745 ep_ring->deq_seg = td->last_trb_seg; in handle_tx_event()
2747 xhci_td_cleanup(xhci, td, ep_ring, td->status); in handle_tx_event()
2759 trb_in_td(xhci, ep_ring->deq_seg, in handle_tx_event()
2760 ep_ring->dequeue, td->last_trb, in handle_tx_event()
2762 return -ESHUTDOWN; in handle_tx_event()
2766 ep_ring->last_td_was_short = true; in handle_tx_event()
2768 ep_ring->last_td_was_short = false; in handle_tx_event()
2770 if (ep->skip) { in handle_tx_event()
2774 ep->skip = false; in handle_tx_event()
2777 ep_trb = &ep_seg->trbs[(ep_trb_dma - ep_seg->dma) / in handle_tx_event()
2784 * No-op TRB could trigger interrupts in a case where in handle_tx_event()
2796 ep_ring->stream_id, in handle_tx_event()
2801 td->status = status; in handle_tx_event()
2804 if (usb_endpoint_xfer_control(&td->urb->ep->desc)) in handle_tx_event()
2806 else if (usb_endpoint_xfer_isoc(&td->urb->ep->desc)) in handle_tx_event()
2811 handling_skipped_tds = ep->skip && in handle_tx_event()
2820 inc_deq(xhci, xhci->event_ring); in handle_tx_event()
2823 * If ep->skip is set, it means there are missed tds on the in handle_tx_event()
2835 xhci->event_ring->deq_seg, in handle_tx_event()
2836 xhci->event_ring->dequeue), in handle_tx_event()
2837 lower_32_bits(le64_to_cpu(event->buffer)), in handle_tx_event()
2838 upper_32_bits(le64_to_cpu(event->buffer)), in handle_tx_event()
2839 le32_to_cpu(event->transfer_len), in handle_tx_event()
2840 le32_to_cpu(event->flags)); in handle_tx_event()
2841 return -ENODEV; in handle_tx_event()
2845 * This function handles all OS-owned events on the event ring. It may drop
2846 * xhci->lock between event processing (e.g. to pass up port status changes).
2857 if (!xhci->event_ring || !xhci->event_ring->dequeue) { in xhci_handle_event()
2859 return -ENOMEM; in xhci_handle_event()
2862 event = xhci->event_ring->dequeue; in xhci_handle_event()
2864 if ((le32_to_cpu(event->event_cmd.flags) & TRB_CYCLE) != in xhci_handle_event()
2865 xhci->event_ring->cycle_state) in xhci_handle_event()
2868 trace_xhci_handle_event(xhci->event_ring, &event->generic); in xhci_handle_event()
2876 switch (le32_to_cpu(event->event_cmd.flags) & TRB_TYPE_BITMASK) { in xhci_handle_event()
2878 handle_cmd_completion(xhci, &event->event_cmd); in xhci_handle_event()
2885 ret = handle_tx_event(xhci, &event->trans_event); in xhci_handle_event()
2893 if ((le32_to_cpu(event->event_cmd.flags) & TRB_TYPE_BITMASK) >= in xhci_handle_event()
2899 le32_to_cpu(event->event_cmd.flags))); in xhci_handle_event()
2901 /* Any of the above functions may drop and re-acquire the lock, so check in xhci_handle_event()
2902 * to make sure a watchdog timer didn't mark the host as non-responsive. in xhci_handle_event()
2904 if (xhci->xhc_state & XHCI_STATE_DYING) { in xhci_handle_event()
2912 inc_deq(xhci, xhci->event_ring); in xhci_handle_event()
2922 * - When all events have finished
2923 * - To avoid "Event Ring Full Error" condition
2931 temp_64 = xhci_read_64(xhci, &xhci->ir_set->erst_dequeue); in xhci_update_erst_dequeue()
2933 if (event_ring_deq != xhci->event_ring->dequeue) { in xhci_update_erst_dequeue()
2934 deq = xhci_trb_virt_to_dma(xhci->event_ring->deq_seg, in xhci_update_erst_dequeue()
2935 xhci->event_ring->dequeue); in xhci_update_erst_dequeue()
2953 xhci_write_64(xhci, temp_64, &xhci->ir_set->erst_dequeue); in xhci_update_erst_dequeue()
2971 spin_lock_irqsave(&xhci->lock, flags); in xhci_irq()
2973 status = readl(&xhci->op_regs->status); in xhci_irq()
2992 * so we can receive interrupts from other MSI-X interrupters. in xhci_irq()
2996 writel(status, &xhci->op_regs->status); in xhci_irq()
2998 if (!hcd->msi_enabled) { in xhci_irq()
3000 irq_pending = readl(&xhci->ir_set->irq_pending); in xhci_irq()
3002 writel(irq_pending, &xhci->ir_set->irq_pending); in xhci_irq()
3005 if (xhci->xhc_state & XHCI_STATE_DYING || in xhci_irq()
3006 xhci->xhc_state & XHCI_STATE_HALTED) { in xhci_irq()
3012 temp_64 = xhci_read_64(xhci, &xhci->ir_set->erst_dequeue); in xhci_irq()
3014 &xhci->ir_set->erst_dequeue); in xhci_irq()
3019 event_ring_deq = xhci->event_ring->dequeue; in xhci_irq()
3027 event_ring_deq = xhci->event_ring->dequeue; in xhci_irq()
3036 spin_unlock_irqrestore(&xhci->lock, flags); in xhci_irq()
3061 trb = &ring->enqueue->generic; in queue_trb()
3062 trb->field[0] = cpu_to_le32(field1); in queue_trb()
3063 trb->field[1] = cpu_to_le32(field2); in queue_trb()
3064 trb->field[2] = cpu_to_le32(field3); in queue_trb()
3067 trb->field[3] = cpu_to_le32(field4); in queue_trb()
3091 return -ENOENT; in prepare_ring()
3095 /* XXX not sure if this should be -ENOENT or not */ in prepare_ring()
3096 return -EINVAL; in prepare_ring()
3108 return -EINVAL; in prepare_ring()
3115 if (ep_ring == xhci->cmd_ring) { in prepare_ring()
3117 return -ENOMEM; in prepare_ring()
3122 num_trbs_needed = num_trbs - ep_ring->num_trbs_free; in prepare_ring()
3126 return -ENOMEM; in prepare_ring()
3130 while (trb_is_link(ep_ring->enqueue)) { in prepare_ring()
3135 !(ep_ring->type == TYPE_ISOC && in prepare_ring()
3136 (xhci->quirks & XHCI_AMD_0x96_HOST))) in prepare_ring()
3137 ep_ring->enqueue->link.control &= in prepare_ring()
3140 ep_ring->enqueue->link.control |= in prepare_ring()
3144 ep_ring->enqueue->link.control ^= cpu_to_le32(TRB_CYCLE); in prepare_ring()
3147 if (link_trb_toggles_cycle(ep_ring->enqueue)) in prepare_ring()
3148 ep_ring->cycle_state ^= 1; in prepare_ring()
3150 ep_ring->enq_seg = ep_ring->enq_seg->next; in prepare_ring()
3151 ep_ring->enqueue = ep_ring->enq_seg->trbs; in prepare_ring()
3169 struct xhci_ep_ctx *ep_ctx = xhci_get_ep_ctx(xhci, xdev->out_ctx, ep_index); in prepare_transfer()
3175 return -EINVAL; in prepare_transfer()
3183 urb_priv = urb->hcpriv; in prepare_transfer()
3184 td = &urb_priv->td[td_index]; in prepare_transfer()
3186 INIT_LIST_HEAD(&td->td_list); in prepare_transfer()
3187 INIT_LIST_HEAD(&td->cancelled_td_list); in prepare_transfer()
3190 ret = usb_hcd_link_urb_to_ep(bus_to_hcd(urb->dev->bus), urb); in prepare_transfer()
3195 td->urb = urb; in prepare_transfer()
3197 list_add_tail(&td->td_list, &ep_ring->td_list); in prepare_transfer()
3198 td->start_seg = ep_ring->enq_seg; in prepare_transfer()
3199 td->first_trb = ep_ring->enqueue; in prepare_transfer()
3208 num_trbs = DIV_ROUND_UP(len + (addr & (TRB_MAX_BUFF_SIZE - 1)), in count_trbs()
3218 return count_trbs(urb->transfer_dma, urb->transfer_buffer_length); in count_trbs_needed()
3226 full_len = urb->transfer_buffer_length; in count_sg_trbs_needed()
3228 for_each_sg(urb->sg, sg, urb->num_mapped_sgs, i) { in count_sg_trbs_needed()
3232 full_len -= len; in count_sg_trbs_needed()
3244 addr = (u64) (urb->transfer_dma + urb->iso_frame_desc[i].offset); in count_isoc_trbs_needed()
3245 len = urb->iso_frame_desc[i].length; in count_isoc_trbs_needed()
3252 if (unlikely(running_total != urb->transfer_buffer_length)) in check_trb_math()
3253 dev_err(&urb->dev->dev, "%s - ep %#x - Miscalculated tx length, " in check_trb_math()
3256 urb->ep->desc.bEndpointAddress, in check_trb_math()
3258 urb->transfer_buffer_length, in check_trb_math()
3259 urb->transfer_buffer_length); in check_trb_math()
3272 start_trb->field[3] |= cpu_to_le32(start_cycle); in giveback_first_trb()
3274 start_trb->field[3] &= cpu_to_le32(~TRB_CYCLE); in giveback_first_trb()
3284 xhci_interval = EP_INTERVAL_TO_UFRAMES(le32_to_cpu(ep_ctx->ep_info)); in check_interval()
3285 ep_interval = urb->interval; in check_interval()
3288 if (urb->dev->speed == USB_SPEED_LOW || in check_interval()
3289 urb->dev->speed == USB_SPEED_FULL) in check_interval()
3296 dev_dbg_ratelimited(&urb->dev->dev, in check_interval()
3300 urb->interval = xhci_interval; in check_interval()
3302 if (urb->dev->speed == USB_SPEED_LOW || in check_interval()
3303 urb->dev->speed == USB_SPEED_FULL) in check_interval()
3304 urb->interval /= 8; in check_interval()
3319 ep_ctx = xhci_get_ep_ctx(xhci, xhci->devs[slot_id]->out_ctx, ep_index); in xhci_queue_intr_tx()
3335 * TD size = total_packet_count - packets_transferred
3352 if (xhci->hci_version < 0x100 && !(xhci->quirks & XHCI_MTK_HOST)) in xhci_td_remainder()
3353 return ((td_total_len - transferred) >> 10); in xhci_td_remainder()
3355 /* One TRB with a zero-length data packet. */ in xhci_td_remainder()
3361 if ((xhci->quirks & XHCI_MTK_HOST) && (xhci->hci_version < 0x100)) in xhci_td_remainder()
3364 maxp = usb_endpoint_maxp(&urb->ep->desc); in xhci_td_remainder()
3368 return (total_packet_count - ((transferred + trb_buff_len) / maxp)); in xhci_td_remainder()
3375 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; in xhci_align_td()
3381 max_pkt = usb_endpoint_maxp(&urb->ep->desc); in xhci_align_td()
3393 *trb_buff_len -= unalign; in xhci_align_td()
3403 new_buff_len = max_pkt - (enqd_len % max_pkt); in xhci_align_td()
3405 if (new_buff_len > (urb->transfer_buffer_length - enqd_len)) in xhci_align_td()
3406 new_buff_len = (urb->transfer_buffer_length - enqd_len); in xhci_align_td()
3410 if (urb->num_sgs) { in xhci_align_td()
3411 len = sg_pcopy_to_buffer(urb->sg, urb->num_sgs, in xhci_align_td()
3412 seg->bounce_buf, new_buff_len, enqd_len); in xhci_align_td()
3417 memcpy(seg->bounce_buf, urb->transfer_buffer + enqd_len, new_buff_len); in xhci_align_td()
3420 seg->bounce_dma = dma_map_single(dev, seg->bounce_buf, in xhci_align_td()
3423 seg->bounce_dma = dma_map_single(dev, seg->bounce_buf, in xhci_align_td()
3427 if (dma_mapping_error(dev, seg->bounce_dma)) { in xhci_align_td()
3433 seg->bounce_len = new_buff_len; in xhci_align_td()
3434 seg->bounce_offs = enqd_len; in xhci_align_td()
3441 /* This is very similar to what ehci-q.c qtd_fill() does */
3462 return -EINVAL; in xhci_queue_bulk_tx()
3464 full_len = urb->transfer_buffer_length; in xhci_queue_bulk_tx()
3466 if (urb->num_sgs) { in xhci_queue_bulk_tx()
3467 num_sgs = urb->num_mapped_sgs; in xhci_queue_bulk_tx()
3468 sg = urb->sg; in xhci_queue_bulk_tx()
3474 addr = (u64) urb->transfer_dma; in xhci_queue_bulk_tx()
3477 ret = prepare_transfer(xhci, xhci->devs[slot_id], in xhci_queue_bulk_tx()
3478 ep_index, urb->stream_id, in xhci_queue_bulk_tx()
3483 urb_priv = urb->hcpriv; in xhci_queue_bulk_tx()
3485 /* Deal with URB_ZERO_PACKET - need one more td/trb */ in xhci_queue_bulk_tx()
3486 if (urb->transfer_flags & URB_ZERO_PACKET && urb_priv->num_tds > 1) in xhci_queue_bulk_tx()
3489 td = &urb_priv->td[0]; in xhci_queue_bulk_tx()
3496 start_trb = &ring->enqueue->generic; in xhci_queue_bulk_tx()
3497 start_cycle = ring->cycle_state; in xhci_queue_bulk_tx()
3500 /* Queue the TRBs, even if they are zero-length */ in xhci_queue_bulk_tx()
3510 trb_buff_len = full_len - enqd_len; in xhci_queue_bulk_tx()
3518 field |= ring->cycle_state; in xhci_queue_bulk_tx()
3525 if (trb_is_link(ring->enqueue + 1)) { in xhci_queue_bulk_tx()
3528 ring->enq_seg)) { in xhci_queue_bulk_tx()
3529 send_addr = ring->enq_seg->bounce_dma; in xhci_queue_bulk_tx()
3531 td->bounce_seg = ring->enq_seg; in xhci_queue_bulk_tx()
3539 td->last_trb = ring->enqueue; in xhci_queue_bulk_tx()
3540 td->last_trb_seg = ring->enq_seg; in xhci_queue_bulk_tx()
3542 memcpy(&send_addr, urb->transfer_buffer, in xhci_queue_bulk_tx()
3566 td->num_trbs++; in xhci_queue_bulk_tx()
3572 --num_sgs; in xhci_queue_bulk_tx()
3573 sent_len -= block_len; in xhci_queue_bulk_tx()
3581 block_len -= sent_len; in xhci_queue_bulk_tx()
3586 ret = prepare_transfer(xhci, xhci->devs[slot_id], in xhci_queue_bulk_tx()
3587 ep_index, urb->stream_id, in xhci_queue_bulk_tx()
3589 urb_priv->td[1].last_trb = ring->enqueue; in xhci_queue_bulk_tx()
3590 urb_priv->td[1].last_trb_seg = ring->enq_seg; in xhci_queue_bulk_tx()
3591 field = TRB_TYPE(TRB_NORMAL) | ring->cycle_state | TRB_IOC; in xhci_queue_bulk_tx()
3593 urb_priv->td[1].num_trbs++; in xhci_queue_bulk_tx()
3597 giveback_first_trb(xhci, slot_id, ep_index, urb->stream_id, in xhci_queue_bulk_tx()
3602 /* Caller must have locked xhci->lock */
3618 return -EINVAL; in xhci_queue_ctrl_tx()
3624 if (!urb->setup_packet) in xhci_queue_ctrl_tx()
3625 return -EINVAL; in xhci_queue_ctrl_tx()
3630 * Don't need to check if we need additional event data and normal TRBs, in xhci_queue_ctrl_tx()
3634 if (urb->transfer_buffer_length > 0) in xhci_queue_ctrl_tx()
3636 ret = prepare_transfer(xhci, xhci->devs[slot_id], in xhci_queue_ctrl_tx()
3637 ep_index, urb->stream_id, in xhci_queue_ctrl_tx()
3642 urb_priv = urb->hcpriv; in xhci_queue_ctrl_tx()
3643 td = &urb_priv->td[0]; in xhci_queue_ctrl_tx()
3644 td->num_trbs = num_trbs; in xhci_queue_ctrl_tx()
3651 start_trb = &ep_ring->enqueue->generic; in xhci_queue_ctrl_tx()
3652 start_cycle = ep_ring->cycle_state; in xhci_queue_ctrl_tx()
3654 /* Queue setup TRB - see section 6.4.1.2.1 */ in xhci_queue_ctrl_tx()
3656 setup = (struct usb_ctrlrequest *) urb->setup_packet; in xhci_queue_ctrl_tx()
3663 if ((xhci->hci_version >= 0x100) || (xhci->quirks & XHCI_MTK_HOST)) { in xhci_queue_ctrl_tx()
3664 if (urb->transfer_buffer_length > 0) { in xhci_queue_ctrl_tx()
3665 if (setup->bRequestType & USB_DIR_IN) in xhci_queue_ctrl_tx()
3673 setup->bRequestType | setup->bRequest << 8 | le16_to_cpu(setup->wValue) << 16, in xhci_queue_ctrl_tx()
3674 le16_to_cpu(setup->wIndex) | le16_to_cpu(setup->wLength) << 16, in xhci_queue_ctrl_tx()
3686 if (urb->transfer_buffer_length > 0) { in xhci_queue_ctrl_tx()
3691 memcpy(&addr, urb->transfer_buffer, in xhci_queue_ctrl_tx()
3692 urb->transfer_buffer_length); in xhci_queue_ctrl_tx()
3696 addr = (u64) urb->transfer_dma; in xhci_queue_ctrl_tx()
3700 urb->transfer_buffer_length, in xhci_queue_ctrl_tx()
3701 urb->transfer_buffer_length, in xhci_queue_ctrl_tx()
3703 length_field = TRB_LEN(urb->transfer_buffer_length) | in xhci_queue_ctrl_tx()
3706 if (setup->bRequestType & USB_DIR_IN) in xhci_queue_ctrl_tx()
3712 field | ep_ring->cycle_state); in xhci_queue_ctrl_tx()
3716 td->last_trb = ep_ring->enqueue; in xhci_queue_ctrl_tx()
3717 td->last_trb_seg = ep_ring->enq_seg; in xhci_queue_ctrl_tx()
3719 /* Queue status TRB - see Table 7 and sections 4.11.2.2 and 6.4.1.2.3 */ in xhci_queue_ctrl_tx()
3721 if (urb->transfer_buffer_length > 0 && setup->bRequestType & USB_DIR_IN) in xhci_queue_ctrl_tx()
3730 field | TRB_IOC | TRB_TYPE(TRB_STATUS) | ep_ring->cycle_state); in xhci_queue_ctrl_tx()
3750 if (xhci->hci_version < 0x100 || urb->dev->speed < USB_SPEED_SUPER) in xhci_get_burst_count()
3753 max_burst = urb->ep->ss_ep_comp.bMaxBurst; in xhci_get_burst_count()
3754 return DIV_ROUND_UP(total_packet_count, max_burst + 1) - 1; in xhci_get_burst_count()
3771 if (xhci->hci_version < 0x100) in xhci_get_last_burst_packet_count()
3774 if (urb->dev->speed >= USB_SPEED_SUPER) { in xhci_get_last_burst_packet_count()
3776 max_burst = urb->ep->ss_ep_comp.bMaxBurst; in xhci_get_last_burst_packet_count()
3779 * number of packets, but the TLBPC field is zero-based. in xhci_get_last_burst_packet_count()
3783 return residue - 1; in xhci_get_last_burst_packet_count()
3787 return total_packet_count - 1; in xhci_get_last_burst_packet_count()
3803 if (urb->dev->speed == USB_SPEED_LOW || in xhci_get_isoc_frame_id()
3804 urb->dev->speed == USB_SPEED_FULL) in xhci_get_isoc_frame_id()
3805 start_frame = urb->start_frame + index * urb->interval; in xhci_get_isoc_frame_id()
3807 start_frame = (urb->start_frame + index * urb->interval) >> 3; in xhci_get_isoc_frame_id()
3817 ist = HCS_IST(xhci->hcs_params2) & 0x7; in xhci_get_isoc_frame_id()
3818 if (HCS_IST(xhci->hcs_params2) & (1 << 3)) in xhci_get_isoc_frame_id()
3834 current_frame_id = readl(&xhci->run_regs->microframe_index); in xhci_get_isoc_frame_id()
3843 __func__, index, readl(&xhci->run_regs->microframe_index), in xhci_get_isoc_frame_id()
3849 ret = -EINVAL; in xhci_get_isoc_frame_id()
3853 ret = -EINVAL; in xhci_get_isoc_frame_id()
3855 ret = -EINVAL; in xhci_get_isoc_frame_id()
3859 if (ret == -EINVAL || start_frame == start_frame_id) { in xhci_get_isoc_frame_id()
3861 if (urb->dev->speed == USB_SPEED_LOW || in xhci_get_isoc_frame_id()
3862 urb->dev->speed == USB_SPEED_FULL) in xhci_get_isoc_frame_id()
3863 urb->start_frame = start_frame; in xhci_get_isoc_frame_id()
3865 urb->start_frame = start_frame << 3; in xhci_get_isoc_frame_id()
3884 if (xhci->hci_version < 0x100) in trb_block_event_intr()
3887 if (i == num_tds - 1) in trb_block_event_intr()
3893 if (i && xhci->quirks & XHCI_AVOID_BEI) in trb_block_event_intr()
3918 xep = &xhci->devs[slot_id]->eps[ep_index]; in xhci_queue_isoc_tx()
3919 ep_ring = xhci->devs[slot_id]->eps[ep_index].ring; in xhci_queue_isoc_tx()
3921 num_tds = urb->number_of_packets; in xhci_queue_isoc_tx()
3924 return -EINVAL; in xhci_queue_isoc_tx()
3926 start_addr = (u64) urb->transfer_dma; in xhci_queue_isoc_tx()
3927 start_trb = &ep_ring->enqueue->generic; in xhci_queue_isoc_tx()
3928 start_cycle = ep_ring->cycle_state; in xhci_queue_isoc_tx()
3930 urb_priv = urb->hcpriv; in xhci_queue_isoc_tx()
3931 /* Queue the TRBs for each TD, even if they are zero-length */ in xhci_queue_isoc_tx()
3939 addr = start_addr + urb->iso_frame_desc[i].offset; in xhci_queue_isoc_tx()
3940 td_len = urb->iso_frame_desc[i].length; in xhci_queue_isoc_tx()
3942 max_pkt = usb_endpoint_maxp(&urb->ep->desc); in xhci_queue_isoc_tx()
3945 /* A zero-length transfer still involves at least one packet. */ in xhci_queue_isoc_tx()
3954 ret = prepare_transfer(xhci, xhci->devs[slot_id], ep_index, in xhci_queue_isoc_tx()
3955 urb->stream_id, trbs_per_td, urb, i, mem_flags); in xhci_queue_isoc_tx()
3961 td = &urb_priv->td[i]; in xhci_queue_isoc_tx()
3962 td->num_trbs = trbs_per_td; in xhci_queue_isoc_tx()
3965 if (!(urb->transfer_flags & URB_ISO_ASAP) && in xhci_queue_isoc_tx()
3966 HCC_CFC(xhci->hcc_params)) { in xhci_queue_isoc_tx()
3979 (i ? ep_ring->cycle_state : !start_cycle); in xhci_queue_isoc_tx()
3982 if (!xep->use_extended_tbc) in xhci_queue_isoc_tx()
3992 ep_ring->cycle_state; in xhci_queue_isoc_tx()
3999 if (j < trbs_per_td - 1) { in xhci_queue_isoc_tx()
4004 td->last_trb = ep_ring->enqueue; in xhci_queue_isoc_tx()
4005 td->last_trb_seg = ep_ring->enq_seg; in xhci_queue_isoc_tx()
4024 if (first_trb && xep->use_extended_tbc) in xhci_queue_isoc_tx()
4038 td_remain_len -= trb_buff_len; in xhci_queue_isoc_tx()
4044 ret = -EINVAL; in xhci_queue_isoc_tx()
4050 if (HCC_CFC(xhci->hcc_params)) in xhci_queue_isoc_tx()
4051 xep->next_frame_id = urb->start_frame + num_tds * urb->interval; in xhci_queue_isoc_tx()
4053 if (xhci_to_hcd(xhci)->self.bandwidth_isoc_reqs == 0) { in xhci_queue_isoc_tx()
4054 if (xhci->quirks & XHCI_AMD_PLL_FIX) in xhci_queue_isoc_tx()
4057 xhci_to_hcd(xhci)->self.bandwidth_isoc_reqs++; in xhci_queue_isoc_tx()
4059 giveback_first_trb(xhci, slot_id, ep_index, urb->stream_id, in xhci_queue_isoc_tx()
4065 for (i--; i >= 0; i--) in xhci_queue_isoc_tx()
4066 list_del_init(&urb_priv->td[i].td_list); in xhci_queue_isoc_tx()
4069 * into No-ops with a software-owned cycle bit. That way the hardware in xhci_queue_isoc_tx()
4071 * overwrite them. td->first_trb and td->start_seg are already set. in xhci_queue_isoc_tx()
4073 urb_priv->td[0].last_trb = ep_ring->enqueue; in xhci_queue_isoc_tx()
4075 td_to_noop(xhci, ep_ring, &urb_priv->td[0], true); in xhci_queue_isoc_tx()
4078 ep_ring->enqueue = urb_priv->td[0].first_trb; in xhci_queue_isoc_tx()
4079 ep_ring->enq_seg = urb_priv->td[0].start_seg; in xhci_queue_isoc_tx()
4080 ep_ring->cycle_state = start_cycle; in xhci_queue_isoc_tx()
4081 ep_ring->num_trbs_free = ep_ring->num_trbs_free_temp; in xhci_queue_isoc_tx()
4082 usb_hcd_unlink_urb_from_ep(bus_to_hcd(urb->dev->bus), urb); in xhci_queue_isoc_tx()
4090 * update urb->start_frame if URB_ISO_ASAP is set in transfer_flags or
4105 xdev = xhci->devs[slot_id]; in xhci_queue_isoc_tx_prepare()
4106 xep = &xhci->devs[slot_id]->eps[ep_index]; in xhci_queue_isoc_tx_prepare()
4107 ep_ring = xdev->eps[ep_index].ring; in xhci_queue_isoc_tx_prepare()
4108 ep_ctx = xhci_get_ep_ctx(xhci, xdev->out_ctx, ep_index); in xhci_queue_isoc_tx_prepare()
4111 num_tds = urb->number_of_packets; in xhci_queue_isoc_tx_prepare()
4129 /* Calculate the start frame and put it in urb->start_frame. */ in xhci_queue_isoc_tx_prepare()
4130 if (HCC_CFC(xhci->hcc_params) && !list_empty(&ep_ring->td_list)) { in xhci_queue_isoc_tx_prepare()
4132 urb->start_frame = xep->next_frame_id; in xhci_queue_isoc_tx_prepare()
4137 start_frame = readl(&xhci->run_regs->microframe_index); in xhci_queue_isoc_tx_prepare()
4143 ist = HCS_IST(xhci->hcs_params2) & 0x7; in xhci_queue_isoc_tx_prepare()
4144 if (HCS_IST(xhci->hcs_params2) & (1 << 3)) in xhci_queue_isoc_tx_prepare()
4153 if (urb->dev->speed == USB_SPEED_LOW || in xhci_queue_isoc_tx_prepare()
4154 urb->dev->speed == USB_SPEED_FULL) { in xhci_queue_isoc_tx_prepare()
4155 start_frame = roundup(start_frame, urb->interval << 3); in xhci_queue_isoc_tx_prepare()
4156 urb->start_frame = start_frame >> 3; in xhci_queue_isoc_tx_prepare()
4158 start_frame = roundup(start_frame, urb->interval); in xhci_queue_isoc_tx_prepare()
4159 urb->start_frame = start_frame; in xhci_queue_isoc_tx_prepare()
4163 ep_ring->num_trbs_free_temp = ep_ring->num_trbs_free; in xhci_queue_isoc_tx_prepare()
4175 * Don't decrement xhci->cmd_ring_reserved_trbs after we've queued the TRB
4182 int reserved_trbs = xhci->cmd_ring_reserved_trbs; in queue_command()
4185 if ((xhci->xhc_state & XHCI_STATE_DYING) || in queue_command()
4186 (xhci->xhc_state & XHCI_STATE_HALTED)) { in queue_command()
4188 return -ESHUTDOWN; in queue_command()
4194 ret = prepare_ring(xhci, xhci->cmd_ring, EP_STATE_RUNNING, in queue_command()
4204 cmd->command_trb = xhci->cmd_ring->enqueue; in queue_command()
4207 if (list_empty(&xhci->cmd_list)) { in queue_command()
4208 xhci->current_cmd = cmd; in queue_command()
4212 list_add_tail(&cmd->cmd_list, &xhci->cmd_list); in queue_command()
4214 queue_trb(xhci, xhci->cmd_ring, false, field1, field2, field3, in queue_command()
4215 field4 | xhci->cmd_ring->cycle_state); in queue_command()
4297 u32 trb_stream_id = STREAM_ID_FOR_TRB(deq_state->stream_id); in xhci_queue_new_dequeue_state()
4306 deq_state->new_deq_seg, in xhci_queue_new_dequeue_state()
4307 (unsigned long long)deq_state->new_deq_seg->dma, in xhci_queue_new_dequeue_state()
4308 deq_state->new_deq_ptr, in xhci_queue_new_dequeue_state()
4310 deq_state->new_deq_seg, deq_state->new_deq_ptr), in xhci_queue_new_dequeue_state()
4311 deq_state->new_cycle_state); in xhci_queue_new_dequeue_state()
4313 addr = xhci_trb_virt_to_dma(deq_state->new_deq_seg, in xhci_queue_new_dequeue_state()
4314 deq_state->new_deq_ptr); in xhci_queue_new_dequeue_state()
4318 deq_state->new_deq_seg, deq_state->new_deq_ptr); in xhci_queue_new_dequeue_state()
4321 ep = &xhci->devs[slot_id]->eps[ep_index]; in xhci_queue_new_dequeue_state()
4322 if ((ep->ep_state & SET_DEQ_PENDING)) { in xhci_queue_new_dequeue_state()
4333 ep->queued_deq_seg = deq_state->new_deq_seg; in xhci_queue_new_dequeue_state()
4334 ep->queued_deq_ptr = deq_state->new_deq_ptr; in xhci_queue_new_dequeue_state()
4335 if (deq_state->stream_id) in xhci_queue_new_dequeue_state()
4338 lower_32_bits(addr) | trb_sct | deq_state->new_cycle_state, in xhci_queue_new_dequeue_state()
4351 ep->ep_state |= SET_DEQ_PENDING; in xhci_queue_new_dequeue_state()