Lines Matching +full:ring +full:- +full:disable +full:- +full:pullup
1 // SPDX-License-Identifier: GPL-2.0
3 * gadget.c - DesignWare USB3 DRD Controller Gadget Framework Link
5 * Copyright (C) 2010-2011 Texas Instruments Incorporated - https://www.ti.com
20 #include <linux/dma-mapping.h>
30 #define DWC3_ALIGN_FRAME(d, n) (((d)->frame_number + ((d)->interval * (n))) \
31 & ~((d)->interval - 1))
34 * dwc3_gadget_set_test_mode - enables usb2 test modes
39 * success or -EINVAL if wrong Test Selector is passed.
45 reg = dwc3_readl(dwc->regs, DWC3_DCTL); in dwc3_gadget_set_test_mode()
57 return -EINVAL; in dwc3_gadget_set_test_mode()
66 * dwc3_gadget_get_link_state - gets current state of usb link
70 * return the link state on success (>= 0) or -ETIMEDOUT.
76 reg = dwc3_readl(dwc->regs, DWC3_DSTS); in dwc3_gadget_get_link_state()
82 * dwc3_gadget_set_link_state - sets usb link to a particular state
87 * return 0 on success or -ETIMEDOUT.
99 while (--retries) { in dwc3_gadget_set_link_state()
100 reg = dwc3_readl(dwc->regs, DWC3_DSTS); in dwc3_gadget_set_link_state()
108 return -ETIMEDOUT; in dwc3_gadget_set_link_state()
111 reg = dwc3_readl(dwc->regs, DWC3_DCTL); in dwc3_gadget_set_link_state()
115 dwc3_writel(dwc->regs, DWC3_DCTL, reg); in dwc3_gadget_set_link_state()
119 dwc3_writel(dwc->regs, DWC3_DCTL, reg); in dwc3_gadget_set_link_state()
130 while (--retries) { in dwc3_gadget_set_link_state()
131 reg = dwc3_readl(dwc->regs, DWC3_DSTS); in dwc3_gadget_set_link_state()
139 return -ETIMEDOUT; in dwc3_gadget_set_link_state()
143 * dwc3_ep_inc_trb - increment a trb index.
153 if (*index == (DWC3_TRB_NUM - 1)) in dwc3_ep_inc_trb()
158 * dwc3_ep_inc_enq - increment endpoint's enqueue pointer
163 dwc3_ep_inc_trb(&dep->trb_enqueue); in dwc3_ep_inc_enq()
167 * dwc3_ep_inc_deq - increment endpoint's dequeue pointer
172 dwc3_ep_inc_trb(&dep->trb_dequeue); in dwc3_ep_inc_deq()
178 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_del_and_unmap_request()
180 list_del(&req->list); in dwc3_gadget_del_and_unmap_request()
181 req->remaining = 0; in dwc3_gadget_del_and_unmap_request()
182 req->needs_extra_trb = false; in dwc3_gadget_del_and_unmap_request()
183 req->num_trbs = 0; in dwc3_gadget_del_and_unmap_request()
185 if (req->request.status == -EINPROGRESS) in dwc3_gadget_del_and_unmap_request()
186 req->request.status = status; in dwc3_gadget_del_and_unmap_request()
188 if (req->trb) in dwc3_gadget_del_and_unmap_request()
189 usb_gadget_unmap_request_by_dev(dwc->sysdev, in dwc3_gadget_del_and_unmap_request()
190 &req->request, req->direction); in dwc3_gadget_del_and_unmap_request()
192 req->trb = NULL; in dwc3_gadget_del_and_unmap_request()
195 if (dep->number > 1) in dwc3_gadget_del_and_unmap_request()
196 pm_runtime_put(dwc->dev); in dwc3_gadget_del_and_unmap_request()
200 * dwc3_gadget_giveback - call struct usb_request's ->complete callback
206 * function will unmap @req and call its ->complete() callback to notify upper
212 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_giveback()
215 req->status = DWC3_REQUEST_STATUS_COMPLETED; in dwc3_gadget_giveback()
217 spin_unlock(&dwc->lock); in dwc3_gadget_giveback()
218 usb_gadget_giveback_request(&dep->endpoint, &req->request); in dwc3_gadget_giveback()
219 spin_lock(&dwc->lock); in dwc3_gadget_giveback()
223 * dwc3_send_gadget_generic_command - issue a generic command for the controller
239 dwc3_writel(dwc->regs, DWC3_DGCMDPAR, param); in dwc3_send_gadget_generic_command()
240 dwc3_writel(dwc->regs, DWC3_DGCMD, cmd | DWC3_DGCMD_CMDACT); in dwc3_send_gadget_generic_command()
243 reg = dwc3_readl(dwc->regs, DWC3_DGCMD); in dwc3_send_gadget_generic_command()
247 ret = -EINVAL; in dwc3_send_gadget_generic_command()
250 } while (--timeout); in dwc3_send_gadget_generic_command()
253 ret = -ETIMEDOUT; in dwc3_send_gadget_generic_command()
254 status = -ETIMEDOUT; in dwc3_send_gadget_generic_command()
265 * dwc3_send_gadget_ep_cmd - issue an endpoint command
276 const struct usb_endpoint_descriptor *desc = dep->endpoint.desc; in dwc3_send_gadget_ep_cmd()
277 struct dwc3 *dwc = dep->dwc; in dwc3_send_gadget_ep_cmd()
283 int ret = -EINVAL; in dwc3_send_gadget_ep_cmd()
295 if (dwc->gadget->speed <= USB_SPEED_HIGH || in dwc3_send_gadget_ep_cmd()
297 reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0)); in dwc3_send_gadget_ep_cmd()
309 dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg); in dwc3_send_gadget_ep_cmd()
324 if (dwc->gadget->speed >= USB_SPEED_SUPER) in dwc3_send_gadget_ep_cmd()
330 dev_WARN_ONCE(dwc->dev, ret, "wakeup failed --> %d\n", in dwc3_send_gadget_ep_cmd()
336 dwc3_writel(dep->regs, DWC3_DEPCMDPAR0, params->param0); in dwc3_send_gadget_ep_cmd()
337 dwc3_writel(dep->regs, DWC3_DEPCMDPAR1, params->param1); in dwc3_send_gadget_ep_cmd()
338 dwc3_writel(dep->regs, DWC3_DEPCMDPAR2, params->param2); in dwc3_send_gadget_ep_cmd()
361 dwc3_writel(dep->regs, DWC3_DEPCMD, cmd); in dwc3_send_gadget_ep_cmd()
363 reg = dwc3_readl(dep->regs, DWC3_DEPCMD); in dwc3_send_gadget_ep_cmd()
372 dev_WARN(dwc->dev, "No resource for %s\n", in dwc3_send_gadget_ep_cmd()
373 dep->name); in dwc3_send_gadget_ep_cmd()
374 ret = -EINVAL; in dwc3_send_gadget_ep_cmd()
384 * Instead of always returning -EINVAL, let's in dwc3_send_gadget_ep_cmd()
386 * the case by returning -EAGAIN. in dwc3_send_gadget_ep_cmd()
388 ret = -EAGAIN; in dwc3_send_gadget_ep_cmd()
391 dev_WARN(dwc->dev, "UNKNOWN cmd status\n"); in dwc3_send_gadget_ep_cmd()
396 } while (--timeout); in dwc3_send_gadget_ep_cmd()
399 ret = -ETIMEDOUT; in dwc3_send_gadget_ep_cmd()
400 cmd_status = -ETIMEDOUT; in dwc3_send_gadget_ep_cmd()
407 dep->flags |= DWC3_EP_TRANSFER_STARTED; in dwc3_send_gadget_ep_cmd()
409 if (ret != -ETIMEDOUT) in dwc3_send_gadget_ep_cmd()
414 reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0)); in dwc3_send_gadget_ep_cmd()
416 dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg); in dwc3_send_gadget_ep_cmd()
424 struct dwc3 *dwc = dep->dwc; in dwc3_send_clear_stall_ep_cmd()
432 * some (non-compliant) hosts may not send ACK TPs for pending in dwc3_send_clear_stall_ep_cmd()
436 if (dep->direction && in dwc3_send_clear_stall_ep_cmd()
438 (dwc->gadget->speed >= USB_SPEED_SUPER)) in dwc3_send_clear_stall_ep_cmd()
449 u32 offset = (char *) trb - (char *) dep->trb_pool; in dwc3_trb_dma_offset()
451 return dep->trb_pool_dma + offset; in dwc3_trb_dma_offset()
456 struct dwc3 *dwc = dep->dwc; in dwc3_alloc_trb_pool()
458 if (dep->trb_pool) in dwc3_alloc_trb_pool()
461 dep->trb_pool = dma_alloc_coherent(dwc->sysdev, in dwc3_alloc_trb_pool()
463 &dep->trb_pool_dma, GFP_KERNEL); in dwc3_alloc_trb_pool()
464 if (!dep->trb_pool) { in dwc3_alloc_trb_pool()
465 dev_err(dep->dwc->dev, "failed to allocate trb pool for %s\n", in dwc3_alloc_trb_pool()
466 dep->name); in dwc3_alloc_trb_pool()
467 return -ENOMEM; in dwc3_alloc_trb_pool()
475 struct dwc3 *dwc = dep->dwc; in dwc3_free_trb_pool()
477 dma_free_coherent(dwc->sysdev, sizeof(struct dwc3_trb) * DWC3_TRB_NUM, in dwc3_free_trb_pool()
478 dep->trb_pool, dep->trb_pool_dma); in dwc3_free_trb_pool()
480 dep->trb_pool = NULL; in dwc3_free_trb_pool()
481 dep->trb_pool_dma = 0; in dwc3_free_trb_pool()
497 * dwc3_gadget_start_config - configure ep resources
526 * triggered only when called for EP0-out, which always happens first, and which
537 if (dep->number) in dwc3_gadget_start_config()
542 dwc = dep->dwc; in dwc3_gadget_start_config()
549 struct dwc3_ep *dep = dwc->eps[i]; in dwc3_gadget_start_config()
567 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_set_ep_config()
569 comp_desc = dep->endpoint.comp_desc; in dwc3_gadget_set_ep_config()
570 desc = dep->endpoint.desc; in dwc3_gadget_set_ep_config()
578 if (dwc->gadget->speed >= USB_SPEED_SUPER) { in dwc3_gadget_set_ep_config()
579 u32 burst = dep->endpoint.maxburst; in dwc3_gadget_set_ep_config()
581 params.param0 |= DWC3_DEPCFG_BURST_SIZE(burst - 1); in dwc3_gadget_set_ep_config()
586 params.param2 |= dep->saved_state; in dwc3_gadget_set_ep_config()
591 if (dep->number <= 1 || usb_endpoint_xfer_isoc(desc)) in dwc3_gadget_set_ep_config()
598 dep->stream_capable = true; in dwc3_gadget_set_ep_config()
610 params.param1 |= DWC3_DEPCFG_EP_NUMBER(dep->number); in dwc3_gadget_set_ep_config()
616 if (dep->direction) in dwc3_gadget_set_ep_config()
617 params.param0 |= DWC3_DEPCFG_FIFO_NUMBER(dep->number >> 1); in dwc3_gadget_set_ep_config()
619 if (desc->bInterval) { in dwc3_gadget_set_ep_config()
630 bInterval_m1 = min_t(u8, desc->bInterval - 1, 13); in dwc3_gadget_set_ep_config()
633 dwc->gadget->speed == USB_SPEED_FULL) in dwc3_gadget_set_ep_config()
634 dep->interval = desc->bInterval; in dwc3_gadget_set_ep_config()
636 dep->interval = 1 << (desc->bInterval - 1); in dwc3_gadget_set_ep_config()
648 * __dwc3_gadget_ep_enable - initializes a hw endpoint
657 const struct usb_endpoint_descriptor *desc = dep->endpoint.desc; in __dwc3_gadget_ep_enable()
658 struct dwc3 *dwc = dep->dwc; in __dwc3_gadget_ep_enable()
663 if (!(dep->flags & DWC3_EP_ENABLED)) { in __dwc3_gadget_ep_enable()
673 if (!(dep->flags & DWC3_EP_ENABLED)) { in __dwc3_gadget_ep_enable()
677 dep->type = usb_endpoint_type(desc); in __dwc3_gadget_ep_enable()
678 dep->flags |= DWC3_EP_ENABLED; in __dwc3_gadget_ep_enable()
680 reg = dwc3_readl(dwc->regs, DWC3_DALEPENA); in __dwc3_gadget_ep_enable()
681 reg |= DWC3_DALEPENA_EP(dep->number); in __dwc3_gadget_ep_enable()
682 dwc3_writel(dwc->regs, DWC3_DALEPENA, reg); in __dwc3_gadget_ep_enable()
687 /* Initialize the TRB ring */ in __dwc3_gadget_ep_enable()
688 dep->trb_dequeue = 0; in __dwc3_gadget_ep_enable()
689 dep->trb_enqueue = 0; in __dwc3_gadget_ep_enable()
690 memset(dep->trb_pool, 0, in __dwc3_gadget_ep_enable()
694 trb_st_hw = &dep->trb_pool[0]; in __dwc3_gadget_ep_enable()
696 trb_link = &dep->trb_pool[DWC3_TRB_NUM - 1]; in __dwc3_gadget_ep_enable()
697 trb_link->bpl = lower_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw)); in __dwc3_gadget_ep_enable()
698 trb_link->bph = upper_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw)); in __dwc3_gadget_ep_enable()
699 trb_link->ctrl |= DWC3_TRBCTL_LINK_TRB; in __dwc3_gadget_ep_enable()
700 trb_link->ctrl |= DWC3_TRB_CTRL_HWO; in __dwc3_gadget_ep_enable()
704 * Issue StartTransfer here with no-op TRB so we can always rely on No in __dwc3_gadget_ep_enable()
715 trb = &dep->trb_pool[0]; in __dwc3_gadget_ep_enable()
727 if (dep->stream_capable) { in __dwc3_gadget_ep_enable()
734 * no-op TRB as normal, but end it immediately. As a in __dwc3_gadget_ep_enable()
747 dep->flags |= DWC3_EP_FORCE_RESTART_STREAM; in __dwc3_gadget_ep_enable()
763 /* - giveback all requests to gadget driver */ in dwc3_remove_requests()
764 while (!list_empty(&dep->started_list)) { in dwc3_remove_requests()
765 req = next_request(&dep->started_list); in dwc3_remove_requests()
770 while (!list_empty(&dep->pending_list)) { in dwc3_remove_requests()
771 req = next_request(&dep->pending_list); in dwc3_remove_requests()
776 while (!list_empty(&dep->cancelled_list)) { in dwc3_remove_requests()
777 req = next_request(&dep->cancelled_list); in dwc3_remove_requests()
784 * __dwc3_gadget_ep_disable - disables a hw endpoint
785 * @dep: the endpoint to disable
795 struct dwc3 *dwc = dep->dwc; in __dwc3_gadget_ep_disable()
801 if (dep->flags & DWC3_EP_STALL) in __dwc3_gadget_ep_disable()
804 reg = dwc3_readl(dwc->regs, DWC3_DALEPENA); in __dwc3_gadget_ep_disable()
805 reg &= ~DWC3_DALEPENA_EP(dep->number); in __dwc3_gadget_ep_disable()
806 dwc3_writel(dwc->regs, DWC3_DALEPENA, reg); in __dwc3_gadget_ep_disable()
808 dwc3_remove_requests(dwc, dep, -ESHUTDOWN); in __dwc3_gadget_ep_disable()
810 dep->stream_capable = false; in __dwc3_gadget_ep_disable()
811 dep->type = 0; in __dwc3_gadget_ep_disable()
812 dep->flags = 0; in __dwc3_gadget_ep_disable()
814 /* Clear out the ep descriptors for non-ep0 */ in __dwc3_gadget_ep_disable()
815 if (dep->number > 1) { in __dwc3_gadget_ep_disable()
816 dep->endpoint.comp_desc = NULL; in __dwc3_gadget_ep_disable()
817 dep->endpoint.desc = NULL; in __dwc3_gadget_ep_disable()
823 /* -------------------------------------------------------------------------- */
828 return -EINVAL; in dwc3_gadget_ep0_enable()
833 return -EINVAL; in dwc3_gadget_ep0_disable()
836 /* -------------------------------------------------------------------------- */
846 if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT) { in dwc3_gadget_ep_enable()
848 return -EINVAL; in dwc3_gadget_ep_enable()
851 if (!desc->wMaxPacketSize) { in dwc3_gadget_ep_enable()
853 return -EINVAL; in dwc3_gadget_ep_enable()
857 dwc = dep->dwc; in dwc3_gadget_ep_enable()
859 if (dev_WARN_ONCE(dwc->dev, dep->flags & DWC3_EP_ENABLED, in dwc3_gadget_ep_enable()
861 dep->name)) in dwc3_gadget_ep_enable()
864 spin_lock_irqsave(&dwc->lock, flags); in dwc3_gadget_ep_enable()
866 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_gadget_ep_enable()
880 return -EINVAL; in dwc3_gadget_ep_disable()
884 dwc = dep->dwc; in dwc3_gadget_ep_disable()
886 if (dev_WARN_ONCE(dwc->dev, !(dep->flags & DWC3_EP_ENABLED), in dwc3_gadget_ep_disable()
888 dep->name)) in dwc3_gadget_ep_disable()
891 spin_lock_irqsave(&dwc->lock, flags); in dwc3_gadget_ep_disable()
893 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_gadget_ep_disable()
908 req->direction = dep->direction; in dwc3_gadget_ep_alloc_request()
909 req->epnum = dep->number; in dwc3_gadget_ep_alloc_request()
910 req->dep = dep; in dwc3_gadget_ep_alloc_request()
911 req->status = DWC3_REQUEST_STATUS_UNKNOWN; in dwc3_gadget_ep_alloc_request()
915 return &req->request; in dwc3_gadget_ep_alloc_request()
928 * dwc3_ep_prev_trb - returns the previous TRB in the ring
929 * @dep: The endpoint with the TRB ring
930 * @index: The index of the current TRB in the ring
941 tmp = DWC3_TRB_NUM - 1; in dwc3_ep_prev_trb()
943 return &dep->trb_pool[tmp - 1]; in dwc3_ep_prev_trb()
951 * If the enqueue & dequeue are equal then the TRB ring is either full in dwc3_calc_trbs_left()
952 * or empty. It's considered full when there are DWC3_TRB_NUM-1 of TRBs in dwc3_calc_trbs_left()
955 if (dep->trb_enqueue == dep->trb_dequeue) { in dwc3_calc_trbs_left()
960 if (!list_empty(&dep->started_list)) in dwc3_calc_trbs_left()
963 return DWC3_TRB_NUM - 1; in dwc3_calc_trbs_left()
966 trbs_left = dep->trb_dequeue - dep->trb_enqueue; in dwc3_calc_trbs_left()
967 trbs_left &= (DWC3_TRB_NUM - 1); in dwc3_calc_trbs_left()
969 if (dep->trb_dequeue < dep->trb_enqueue) in dwc3_calc_trbs_left()
970 trbs_left--; in dwc3_calc_trbs_left()
976 * dwc3_prepare_one_trb - setup one TRB from one request
992 unsigned int stream_id = req->request.stream_id; in dwc3_prepare_one_trb()
993 unsigned int short_not_ok = req->request.short_not_ok; in dwc3_prepare_one_trb()
994 unsigned int no_interrupt = req->request.no_interrupt; in dwc3_prepare_one_trb()
995 unsigned int is_last = req->request.is_last; in dwc3_prepare_one_trb()
996 struct dwc3 *dwc = dep->dwc; in dwc3_prepare_one_trb()
997 struct usb_gadget *gadget = dwc->gadget; in dwc3_prepare_one_trb()
998 enum usb_device_speed speed = gadget->speed; in dwc3_prepare_one_trb()
1001 dma = dep->dwc->bounce_addr; in dwc3_prepare_one_trb()
1002 else if (req->request.num_sgs > 0) in dwc3_prepare_one_trb()
1003 dma = sg_dma_address(req->start_sg); in dwc3_prepare_one_trb()
1005 dma = req->request.dma; in dwc3_prepare_one_trb()
1007 trb = &dep->trb_pool[dep->trb_enqueue]; in dwc3_prepare_one_trb()
1009 if (!req->trb) { in dwc3_prepare_one_trb()
1011 req->trb = trb; in dwc3_prepare_one_trb()
1012 req->trb_dma = dwc3_trb_dma_offset(dep, trb); in dwc3_prepare_one_trb()
1015 req->num_trbs++; in dwc3_prepare_one_trb()
1017 trb->size = DWC3_TRB_SIZE_LENGTH(trb_length); in dwc3_prepare_one_trb()
1018 trb->bpl = lower_32_bits(dma); in dwc3_prepare_one_trb()
1019 trb->bph = upper_32_bits(dma); in dwc3_prepare_one_trb()
1021 switch (usb_endpoint_type(dep->endpoint.desc)) { in dwc3_prepare_one_trb()
1023 trb->ctrl = DWC3_TRBCTL_CONTROL_SETUP; in dwc3_prepare_one_trb()
1028 trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS_FIRST; in dwc3_prepare_one_trb()
1044 * - DATA0 in dwc3_prepare_one_trb()
1047 * - DATA1, DATA0 in dwc3_prepare_one_trb()
1050 * - DATA2, DATA1, DATA0 in dwc3_prepare_one_trb()
1053 struct usb_ep *ep = &dep->endpoint; in dwc3_prepare_one_trb()
1055 unsigned int maxp = usb_endpoint_maxp(ep->desc); in dwc3_prepare_one_trb()
1057 if (req->request.length <= (2 * maxp)) in dwc3_prepare_one_trb()
1058 mult--; in dwc3_prepare_one_trb()
1060 if (req->request.length <= maxp) in dwc3_prepare_one_trb()
1061 mult--; in dwc3_prepare_one_trb()
1063 trb->size |= DWC3_TRB_SIZE_PCM1(mult); in dwc3_prepare_one_trb()
1066 trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS; in dwc3_prepare_one_trb()
1070 trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI; in dwc3_prepare_one_trb()
1075 trb->ctrl = DWC3_TRBCTL_NORMAL; in dwc3_prepare_one_trb()
1082 dev_WARN(dwc->dev, "Unknown endpoint type %d\n", in dwc3_prepare_one_trb()
1083 usb_endpoint_type(dep->endpoint.desc)); in dwc3_prepare_one_trb()
1090 if (usb_endpoint_dir_out(dep->endpoint.desc)) { in dwc3_prepare_one_trb()
1091 if (!dep->stream_capable) in dwc3_prepare_one_trb()
1092 trb->ctrl |= DWC3_TRB_CTRL_CSP; in dwc3_prepare_one_trb()
1095 trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI; in dwc3_prepare_one_trb()
1099 trb->ctrl |= DWC3_TRB_CTRL_IOC; in dwc3_prepare_one_trb()
1102 trb->ctrl |= DWC3_TRB_CTRL_CHN; in dwc3_prepare_one_trb()
1103 else if (dep->stream_capable && is_last) in dwc3_prepare_one_trb()
1104 trb->ctrl |= DWC3_TRB_CTRL_LST; in dwc3_prepare_one_trb()
1106 if (usb_endpoint_xfer_bulk(dep->endpoint.desc) && dep->stream_capable) in dwc3_prepare_one_trb()
1107 trb->ctrl |= DWC3_TRB_CTRL_SID_SOFN(stream_id); in dwc3_prepare_one_trb()
1117 * However there is a possibility of CPU re-ordering here which can cause in dwc3_prepare_one_trb()
1119 * Add a write memory barrier to prevent CPU re-ordering. in dwc3_prepare_one_trb()
1122 trb->ctrl |= DWC3_TRB_CTRL_HWO; in dwc3_prepare_one_trb()
1131 unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc); in dwc3_needs_extra_trb()
1132 unsigned int rem = req->request.length % maxp; in dwc3_needs_extra_trb()
1134 if ((req->request.length && req->request.zero && !rem && in dwc3_needs_extra_trb()
1135 !usb_endpoint_xfer_isoc(dep->endpoint.desc)) || in dwc3_needs_extra_trb()
1136 (!req->direction && rem)) in dwc3_needs_extra_trb()
1143 * dwc3_prepare_last_sg - prepare TRBs for the last SG entry
1155 unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc); in dwc3_prepare_last_sg()
1156 unsigned int rem = req->request.length % maxp; in dwc3_prepare_last_sg()
1165 req->needs_extra_trb = num_trbs > 1; in dwc3_prepare_last_sg()
1168 if (req->direction || req->request.length) in dwc3_prepare_last_sg()
1170 req->needs_extra_trb, node, false, false); in dwc3_prepare_last_sg()
1173 if ((!req->direction && !req->request.length) || req->needs_extra_trb) in dwc3_prepare_last_sg()
1175 req->direction ? 0 : maxp - rem, in dwc3_prepare_last_sg()
1184 struct scatterlist *sg = req->start_sg; in dwc3_prepare_trbs_sg()
1187 unsigned int length = req->request.length; in dwc3_prepare_trbs_sg()
1188 unsigned int remaining = req->num_pending_sgs; in dwc3_prepare_trbs_sg()
1189 unsigned int num_queued_sgs = req->request.num_mapped_sgs - remaining; in dwc3_prepare_trbs_sg()
1190 unsigned int num_trbs = req->num_trbs; in dwc3_prepare_trbs_sg()
1197 for_each_sg(req->request.sg, s, num_queued_sgs, i) in dwc3_prepare_trbs_sg()
1198 length -= sg_dma_len(s); in dwc3_prepare_trbs_sg()
1208 length -= trb_length; in dwc3_prepare_trbs_sg()
1217 if ((i == remaining - 1) || !length) in dwc3_prepare_trbs_sg()
1250 req->start_sg = sg_next(s); in dwc3_prepare_trbs_sg()
1252 req->num_queued_sgs++; in dwc3_prepare_trbs_sg()
1253 req->num_pending_sgs--; in dwc3_prepare_trbs_sg()
1261 req->num_pending_sgs = 0; in dwc3_prepare_trbs_sg()
1269 return req->num_trbs - num_trbs; in dwc3_prepare_trbs_sg()
1275 return dwc3_prepare_last_sg(dep, req, req->request.length, 0); in dwc3_prepare_trbs_linear()
1279 * dwc3_prepare_trbs - setup TRBs from requests
1305 list_for_each_entry(req, &dep->started_list, list) { in dwc3_prepare_trbs()
1306 if (req->num_pending_sgs > 0) { in dwc3_prepare_trbs()
1308 if (!ret || req->num_pending_sgs) in dwc3_prepare_trbs()
1320 if (dep->stream_capable && req->request.is_last) in dwc3_prepare_trbs()
1324 list_for_each_entry_safe(req, n, &dep->pending_list, list) { in dwc3_prepare_trbs()
1325 struct dwc3 *dwc = dep->dwc; in dwc3_prepare_trbs()
1327 ret = usb_gadget_map_request_by_dev(dwc->sysdev, &req->request, in dwc3_prepare_trbs()
1328 dep->direction); in dwc3_prepare_trbs()
1332 req->sg = req->request.sg; in dwc3_prepare_trbs()
1333 req->start_sg = req->sg; in dwc3_prepare_trbs()
1334 req->num_queued_sgs = 0; in dwc3_prepare_trbs()
1335 req->num_pending_sgs = req->request.num_mapped_sgs; in dwc3_prepare_trbs()
1337 if (req->num_pending_sgs > 0) { in dwc3_prepare_trbs()
1339 if (req->num_pending_sgs) in dwc3_prepare_trbs()
1353 if (dep->stream_capable && req->request.is_last) in dwc3_prepare_trbs()
1379 starting = !(dep->flags & DWC3_EP_TRANSFER_STARTED); in __dwc3_gadget_kick_transfer()
1388 req = next_request(&dep->started_list); in __dwc3_gadget_kick_transfer()
1390 dep->flags |= DWC3_EP_PENDING_REQUEST; in __dwc3_gadget_kick_transfer()
1397 params.param0 = upper_32_bits(req->trb_dma); in __dwc3_gadget_kick_transfer()
1398 params.param1 = lower_32_bits(req->trb_dma); in __dwc3_gadget_kick_transfer()
1401 if (dep->stream_capable) in __dwc3_gadget_kick_transfer()
1402 cmd |= DWC3_DEPCMD_PARAM(req->request.stream_id); in __dwc3_gadget_kick_transfer()
1404 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) in __dwc3_gadget_kick_transfer()
1405 cmd |= DWC3_DEPCMD_PARAM(dep->frame_number); in __dwc3_gadget_kick_transfer()
1408 DWC3_DEPCMD_PARAM(dep->resource_index); in __dwc3_gadget_kick_transfer()
1415 if (ret == -EAGAIN) in __dwc3_gadget_kick_transfer()
1420 list_for_each_entry_safe(req, tmp, &dep->started_list, list) in __dwc3_gadget_kick_transfer()
1424 if (!(dep->flags & DWC3_EP_END_TRANSFER_PENDING)) in __dwc3_gadget_kick_transfer()
1430 if (dep->stream_capable && req->request.is_last) in __dwc3_gadget_kick_transfer()
1431 dep->flags |= DWC3_EP_WAIT_TRANSFER_COMPLETE; in __dwc3_gadget_kick_transfer()
1440 reg = dwc3_readl(dwc->regs, DWC3_DSTS); in __dwc3_gadget_get_frame()
1445 * __dwc3_stop_active_transfer - stop the current active transfer
1457 struct dwc3 *dwc = dep->dwc; in __dwc3_stop_active_transfer()
1465 cmd |= DWC3_DEPCMD_PARAM(dep->resource_index); in __dwc3_stop_active_transfer()
1469 dep->resource_index = 0; in __dwc3_stop_active_transfer()
1474 dep->flags &= ~DWC3_EP_TRANSFER_STARTED; in __dwc3_stop_active_transfer()
1476 dep->flags |= DWC3_EP_END_TRANSFER_PENDING; in __dwc3_stop_active_transfer()
1483 * dwc3_gadget_start_isoc_quirk - workaround invalid frame number
1486 * This function tests for the correct combination of BIT[15:14] from the 16-bit
1490 * In DWC_usb31 version 1.70a-ea06 and prior, for highspeed and fullspeed
1491 * isochronous IN, BIT[15:14] of the 16-bit microframe number reported by the
1495 * internal 16-bit microframe, the START TRANSFER command will pass and the
1498 * other conditions, the START TRANSFER command will fail with bus-expiry.
1503 * (or 2 seconds). 4 seconds into the future will result in a bus-expiry status.
1506 * command status will result in a 2-second delay start. The smaller BIT[15:14]
1530 while (dep->combo_num < 2) { in dwc3_gadget_start_isoc_quirk()
1537 * 4 uframes in the future with BIT[15:14] as dep->combo_num in dwc3_gadget_start_isoc_quirk()
1539 test_frame_number = dep->frame_number & DWC3_FRNUMBER_MASK; in dwc3_gadget_start_isoc_quirk()
1540 test_frame_number |= dep->combo_num << 14; in dwc3_gadget_start_isoc_quirk()
1541 test_frame_number += max_t(u32, 4, dep->interval); in dwc3_gadget_start_isoc_quirk()
1543 params.param0 = upper_32_bits(dep->dwc->bounce_addr); in dwc3_gadget_start_isoc_quirk()
1544 params.param1 = lower_32_bits(dep->dwc->bounce_addr); in dwc3_gadget_start_isoc_quirk()
1550 /* Redo if some other failure beside bus-expiry is received */ in dwc3_gadget_start_isoc_quirk()
1551 if (cmd_status && cmd_status != -EAGAIN) { in dwc3_gadget_start_isoc_quirk()
1552 dep->start_cmd_status = 0; in dwc3_gadget_start_isoc_quirk()
1553 dep->combo_num = 0; in dwc3_gadget_start_isoc_quirk()
1558 if (dep->combo_num == 0) in dwc3_gadget_start_isoc_quirk()
1559 dep->start_cmd_status = cmd_status; in dwc3_gadget_start_isoc_quirk()
1561 dep->combo_num++; in dwc3_gadget_start_isoc_quirk()
1574 test0 = (dep->start_cmd_status == 0); in dwc3_gadget_start_isoc_quirk()
1578 dep->combo_num = 1; in dwc3_gadget_start_isoc_quirk()
1580 dep->combo_num = 2; in dwc3_gadget_start_isoc_quirk()
1582 dep->combo_num = 3; in dwc3_gadget_start_isoc_quirk()
1584 dep->combo_num = 0; in dwc3_gadget_start_isoc_quirk()
1586 dep->frame_number &= DWC3_FRNUMBER_MASK; in dwc3_gadget_start_isoc_quirk()
1587 dep->frame_number |= dep->combo_num << 14; in dwc3_gadget_start_isoc_quirk()
1588 dep->frame_number += max_t(u32, 4, dep->interval); in dwc3_gadget_start_isoc_quirk()
1591 dep->start_cmd_status = 0; in dwc3_gadget_start_isoc_quirk()
1592 dep->combo_num = 0; in dwc3_gadget_start_isoc_quirk()
1599 const struct usb_endpoint_descriptor *desc = dep->endpoint.desc; in __dwc3_gadget_start_isoc()
1600 struct dwc3 *dwc = dep->dwc; in __dwc3_gadget_start_isoc()
1604 if (list_empty(&dep->pending_list) && in __dwc3_gadget_start_isoc()
1605 list_empty(&dep->started_list)) { in __dwc3_gadget_start_isoc()
1606 dep->flags |= DWC3_EP_PENDING_REQUEST; in __dwc3_gadget_start_isoc()
1607 return -EAGAIN; in __dwc3_gadget_start_isoc()
1610 if (!dwc->dis_start_transfer_quirk && in __dwc3_gadget_start_isoc()
1613 if (dwc->gadget->speed <= USB_SPEED_HIGH && dep->direction) in __dwc3_gadget_start_isoc()
1617 if (desc->bInterval <= 14 && in __dwc3_gadget_start_isoc()
1618 dwc->gadget->speed >= USB_SPEED_HIGH) { in __dwc3_gadget_start_isoc()
1621 (dep->frame_number & DWC3_FRNUMBER_MASK); in __dwc3_gadget_start_isoc()
1632 dep->frame_number = (dep->frame_number & ~DWC3_FRNUMBER_MASK) | in __dwc3_gadget_start_isoc()
1635 dep->frame_number += BIT(14); in __dwc3_gadget_start_isoc()
1639 dep->frame_number = DWC3_ALIGN_FRAME(dep, i + 1); in __dwc3_gadget_start_isoc()
1642 if (ret != -EAGAIN) in __dwc3_gadget_start_isoc()
1647 * After a number of unsuccessful start attempts due to bus-expiry in __dwc3_gadget_start_isoc()
1651 if (ret == -EAGAIN) in __dwc3_gadget_start_isoc()
1659 struct dwc3 *dwc = dep->dwc; in __dwc3_gadget_ep_queue()
1661 if (!dep->endpoint.desc || !dwc->pullups_connected || !dwc->connected) { in __dwc3_gadget_ep_queue()
1662 dev_err(dwc->dev, "%s: can't queue to disabled endpoint\n", in __dwc3_gadget_ep_queue()
1663 dep->name); in __dwc3_gadget_ep_queue()
1664 return -ESHUTDOWN; in __dwc3_gadget_ep_queue()
1667 if (WARN(req->dep != dep, "request %pK belongs to '%s'\n", in __dwc3_gadget_ep_queue()
1668 &req->request, req->dep->name)) in __dwc3_gadget_ep_queue()
1669 return -EINVAL; in __dwc3_gadget_ep_queue()
1671 if (WARN(req->status < DWC3_REQUEST_STATUS_COMPLETED, in __dwc3_gadget_ep_queue()
1673 dep->name, &req->request)) in __dwc3_gadget_ep_queue()
1674 return -EINVAL; in __dwc3_gadget_ep_queue()
1676 pm_runtime_get(dwc->dev); in __dwc3_gadget_ep_queue()
1678 req->request.actual = 0; in __dwc3_gadget_ep_queue()
1679 req->request.status = -EINPROGRESS; in __dwc3_gadget_ep_queue()
1683 list_add_tail(&req->list, &dep->pending_list); in __dwc3_gadget_ep_queue()
1684 req->status = DWC3_REQUEST_STATUS_QUEUED; in __dwc3_gadget_ep_queue()
1686 if (dep->flags & DWC3_EP_WAIT_TRANSFER_COMPLETE) in __dwc3_gadget_ep_queue()
1693 if ((dep->flags & DWC3_EP_END_TRANSFER_PENDING) || in __dwc3_gadget_ep_queue()
1694 (dep->flags & DWC3_EP_WEDGE) || in __dwc3_gadget_ep_queue()
1695 (dep->flags & DWC3_EP_STALL)) { in __dwc3_gadget_ep_queue()
1696 dep->flags |= DWC3_EP_DELAY_START; in __dwc3_gadget_ep_queue()
1703 * (micro-)frame number. in __dwc3_gadget_ep_queue()
1708 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) { in __dwc3_gadget_ep_queue()
1709 if (!(dep->flags & DWC3_EP_PENDING_REQUEST) && in __dwc3_gadget_ep_queue()
1710 !(dep->flags & DWC3_EP_TRANSFER_STARTED)) in __dwc3_gadget_ep_queue()
1713 if ((dep->flags & DWC3_EP_PENDING_REQUEST)) { in __dwc3_gadget_ep_queue()
1714 if (!(dep->flags & DWC3_EP_TRANSFER_STARTED)) in __dwc3_gadget_ep_queue()
1729 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_ep_queue()
1735 spin_lock_irqsave(&dwc->lock, flags); in dwc3_gadget_ep_queue()
1737 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_gadget_ep_queue()
1746 /* If req->trb is not set, then the request has not started */ in dwc3_gadget_ep_skip_trbs()
1747 if (!req->trb) in dwc3_gadget_ep_skip_trbs()
1760 for (i = 0; i < req->num_trbs; i++) { in dwc3_gadget_ep_skip_trbs()
1763 trb = &dep->trb_pool[dep->trb_dequeue]; in dwc3_gadget_ep_skip_trbs()
1764 trb->ctrl &= ~DWC3_TRB_CTRL_HWO; in dwc3_gadget_ep_skip_trbs()
1768 req->num_trbs = 0; in dwc3_gadget_ep_skip_trbs()
1776 list_for_each_entry_safe(req, tmp, &dep->cancelled_list, list) { in dwc3_gadget_ep_cleanup_cancelled_requests()
1778 dwc3_gadget_giveback(dep, req, -ECONNRESET); in dwc3_gadget_ep_cleanup_cancelled_requests()
1789 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_ep_dequeue()
1796 spin_lock_irqsave(&dwc->lock, flags); in dwc3_gadget_ep_dequeue()
1798 list_for_each_entry(r, &dep->cancelled_list, list) { in dwc3_gadget_ep_dequeue()
1803 list_for_each_entry(r, &dep->pending_list, list) { in dwc3_gadget_ep_dequeue()
1805 dwc3_gadget_giveback(dep, req, -ECONNRESET); in dwc3_gadget_ep_dequeue()
1810 list_for_each_entry(r, &dep->started_list, list) { in dwc3_gadget_ep_dequeue()
1821 list_for_each_entry_safe(r, t, &dep->started_list, list) in dwc3_gadget_ep_dequeue()
1824 dep->flags &= ~DWC3_EP_WAIT_TRANSFER_COMPLETE; in dwc3_gadget_ep_dequeue()
1830 dev_err(dwc->dev, "request %pK was not queued to %s\n", in dwc3_gadget_ep_dequeue()
1831 request, ep->name); in dwc3_gadget_ep_dequeue()
1832 ret = -EINVAL; in dwc3_gadget_ep_dequeue()
1834 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_gadget_ep_dequeue()
1842 struct dwc3 *dwc = dep->dwc; in __dwc3_gadget_ep_set_halt()
1847 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) { in __dwc3_gadget_ep_set_halt()
1848 dev_err(dwc->dev, "%s is of Isochronous type\n", dep->name); in __dwc3_gadget_ep_set_halt()
1849 return -EINVAL; in __dwc3_gadget_ep_set_halt()
1860 if (dep->number > 1) in __dwc3_gadget_ep_set_halt()
1861 trb = dwc3_ep_prev_trb(dep, dep->trb_enqueue); in __dwc3_gadget_ep_set_halt()
1863 trb = &dwc->ep0_trb[dep->trb_enqueue]; in __dwc3_gadget_ep_set_halt()
1865 transfer_in_flight = trb->ctrl & DWC3_TRB_CTRL_HWO; in __dwc3_gadget_ep_set_halt()
1866 started = !list_empty(&dep->started_list); in __dwc3_gadget_ep_set_halt()
1868 if (!protocol && ((dep->direction && transfer_in_flight) || in __dwc3_gadget_ep_set_halt()
1869 (!dep->direction && started))) { in __dwc3_gadget_ep_set_halt()
1870 return -EAGAIN; in __dwc3_gadget_ep_set_halt()
1876 dev_err(dwc->dev, "failed to set STALL on %s\n", in __dwc3_gadget_ep_set_halt()
1877 dep->name); in __dwc3_gadget_ep_set_halt()
1879 dep->flags |= DWC3_EP_STALL; in __dwc3_gadget_ep_set_halt()
1886 if (dep->number <= 1) { in __dwc3_gadget_ep_set_halt()
1887 dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE); in __dwc3_gadget_ep_set_halt()
1893 list_for_each_entry_safe(req, tmp, &dep->started_list, list) in __dwc3_gadget_ep_set_halt()
1896 if (dep->flags & DWC3_EP_END_TRANSFER_PENDING) { in __dwc3_gadget_ep_set_halt()
1897 dep->flags |= DWC3_EP_PENDING_CLEAR_STALL; in __dwc3_gadget_ep_set_halt()
1905 dev_err(dwc->dev, "failed to clear STALL on %s\n", in __dwc3_gadget_ep_set_halt()
1906 dep->name); in __dwc3_gadget_ep_set_halt()
1910 dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE); in __dwc3_gadget_ep_set_halt()
1912 if ((dep->flags & DWC3_EP_DELAY_START) && in __dwc3_gadget_ep_set_halt()
1913 !usb_endpoint_xfer_isoc(dep->endpoint.desc)) in __dwc3_gadget_ep_set_halt()
1916 dep->flags &= ~DWC3_EP_DELAY_START; in __dwc3_gadget_ep_set_halt()
1925 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_ep_set_halt()
1931 spin_lock_irqsave(&dwc->lock, flags); in dwc3_gadget_ep_set_halt()
1933 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_gadget_ep_set_halt()
1941 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_ep_set_wedge()
1945 spin_lock_irqsave(&dwc->lock, flags); in dwc3_gadget_ep_set_wedge()
1946 dep->flags |= DWC3_EP_WEDGE; in dwc3_gadget_ep_set_wedge()
1948 if (dep->number == 0 || dep->number == 1) in dwc3_gadget_ep_set_wedge()
1952 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_gadget_ep_set_wedge()
1957 /* -------------------------------------------------------------------------- */
1967 .disable = dwc3_gadget_ep0_disable,
1978 .disable = dwc3_gadget_ep_disable,
1987 /* -------------------------------------------------------------------------- */
2011 reg = dwc3_readl(dwc->regs, DWC3_DSTS); in __dwc3_gadget_wakeup()
2024 return -EINVAL; in __dwc3_gadget_wakeup()
2029 dev_err(dwc->dev, "failed to put link in Recovery\n"); in __dwc3_gadget_wakeup()
2036 reg = dwc3_readl(dwc->regs, DWC3_DCTL); in __dwc3_gadget_wakeup()
2038 dwc3_writel(dwc->regs, DWC3_DCTL, reg); in __dwc3_gadget_wakeup()
2044 while (retries--) { in __dwc3_gadget_wakeup()
2045 reg = dwc3_readl(dwc->regs, DWC3_DSTS); in __dwc3_gadget_wakeup()
2053 dev_err(dwc->dev, "failed to send remote wakeup\n"); in __dwc3_gadget_wakeup()
2054 return -EINVAL; in __dwc3_gadget_wakeup()
2066 spin_lock_irqsave(&dwc->lock, flags); in dwc3_gadget_wakeup()
2068 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_gadget_wakeup()
2079 spin_lock_irqsave(&dwc->lock, flags); in dwc3_gadget_set_selfpowered()
2080 g->is_selfpowered = !!is_selfpowered; in dwc3_gadget_set_selfpowered()
2081 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_gadget_set_selfpowered()
2090 for (epnum = 2; epnum < dwc->num_eps; epnum++) { in dwc3_stop_active_transfers()
2093 dep = dwc->eps[epnum]; in dwc3_stop_active_transfers()
2097 dwc3_remove_requests(dwc, dep, -ESHUTDOWN); in dwc3_stop_active_transfers()
2106 if (pm_runtime_suspended(dwc->dev)) in dwc3_gadget_run_stop()
2109 reg = dwc3_readl(dwc->regs, DWC3_DCTL); in dwc3_gadget_run_stop()
2120 if (dwc->has_hibernation) in dwc3_gadget_run_stop()
2123 dwc->pullups_connected = true; in dwc3_gadget_run_stop()
2127 if (dwc->has_hibernation && !suspend) in dwc3_gadget_run_stop()
2130 dwc->pullups_connected = false; in dwc3_gadget_run_stop()
2136 reg = dwc3_readl(dwc->regs, DWC3_DSTS); in dwc3_gadget_run_stop()
2138 } while (--timeout && !(!is_on ^ !reg)); in dwc3_gadget_run_stop()
2141 return -ETIMEDOUT; in dwc3_gadget_run_stop()
2154 spin_lock_irqsave(&dwc->lock, flags); in dwc3_gadget_soft_disconnect()
2155 dwc->connected = false; in dwc3_gadget_soft_disconnect()
2159 * Section 4.1.8 Table 4-7, it states that for a device-initiated in dwc3_gadget_soft_disconnect()
2166 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_gadget_soft_disconnect()
2185 dwc->softconnect = is_on; in dwc3_gadget_pullup()
2190 if (!is_on && dwc->ep0state != EP0_SETUP_PHASE) { in dwc3_gadget_pullup()
2191 reinit_completion(&dwc->ep0_in_setup); in dwc3_gadget_pullup()
2193 ret = wait_for_completion_timeout(&dwc->ep0_in_setup, in dwc3_gadget_pullup()
2196 dev_warn(dwc->dev, "timed out waiting for SETUP phase\n"); in dwc3_gadget_pullup()
2205 pm_runtime_barrier(dwc->dev); in dwc3_gadget_pullup()
2206 if (pm_runtime_suspended(dwc->dev)) in dwc3_gadget_pullup()
2215 ret = pm_runtime_get_sync(dwc->dev); in dwc3_gadget_pullup()
2217 pm_runtime_put(dwc->dev); in dwc3_gadget_pullup()
2219 pm_runtime_set_suspended(dwc->dev); in dwc3_gadget_pullup()
2223 if (dwc->pullups_connected == is_on) { in dwc3_gadget_pullup()
2224 pm_runtime_put(dwc->dev); in dwc3_gadget_pullup()
2234 * device-initiated disconnect requires a core soft reset in dwc3_gadget_pullup()
2244 pm_runtime_put(dwc->dev); in dwc3_gadget_pullup()
2266 /* On 2.30a and above this bit enables U3/L2-L1 Suspend Events */ in dwc3_gadget_enable_irq()
2270 dwc3_writel(dwc->regs, DWC3_DEVTEN, reg); in dwc3_gadget_enable_irq()
2276 dwc3_writel(dwc->regs, DWC3_DEVTEN, 0x00); in dwc3_gadget_disable_irq()
2283 * dwc3_gadget_setup_nump - calculate and initialize NUMP field of %DWC3_DCFG
2296 * RxFIFO Size = (RAM2_DEPTH * MDWIDTH / 8) - 24 - 16;
2310 ram2_depth = DWC3_GHWPARAMS7_RAM2_DEPTH(dwc->hwparams.hwparams7); in dwc3_gadget_setup_nump()
2311 mdwidth = DWC3_GHWPARAMS0_MDWIDTH(dwc->hwparams.hwparams0); in dwc3_gadget_setup_nump()
2313 mdwidth += DWC3_GHWPARAMS6_MDWIDTH(dwc->hwparams.hwparams6); in dwc3_gadget_setup_nump()
2315 nump = ((ram2_depth * mdwidth / 8) - 24 - 16) / 1024; in dwc3_gadget_setup_nump()
2319 reg = dwc3_readl(dwc->regs, DWC3_DCFG); in dwc3_gadget_setup_nump()
2322 dwc3_writel(dwc->regs, DWC3_DCFG, reg); in dwc3_gadget_setup_nump()
2332 * Use IMOD if enabled via dwc->imod_interval. Otherwise, if in __dwc3_gadget_start()
2333 * the core supports IMOD, disable it. in __dwc3_gadget_start()
2335 if (dwc->imod_interval) { in __dwc3_gadget_start()
2336 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), dwc->imod_interval); in __dwc3_gadget_start()
2337 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), DWC3_GEVNTCOUNT_EHB); in __dwc3_gadget_start()
2339 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), 0); in __dwc3_gadget_start()
2349 reg = dwc3_readl(dwc->regs, DWC3_GRXTHRCFG); in __dwc3_gadget_start()
2355 dwc3_writel(dwc->regs, DWC3_GRXTHRCFG, reg); in __dwc3_gadget_start()
2362 dep = dwc->eps[0]; in __dwc3_gadget_start()
2365 dev_err(dwc->dev, "failed to enable %s\n", dep->name); in __dwc3_gadget_start()
2369 dep = dwc->eps[1]; in __dwc3_gadget_start()
2372 dev_err(dwc->dev, "failed to enable %s\n", dep->name); in __dwc3_gadget_start()
2377 dwc->ep0state = EP0_SETUP_PHASE; in __dwc3_gadget_start()
2378 dwc->link_state = DWC3_LINK_STATE_SS_DIS; in __dwc3_gadget_start()
2379 dwc->delayed_status = false; in __dwc3_gadget_start()
2387 __dwc3_gadget_ep_disable(dwc->eps[0]); in __dwc3_gadget_start()
2401 irq = dwc->irq_gadget; in dwc3_gadget_start()
2403 IRQF_SHARED, "dwc3", dwc->ev_buf); in dwc3_gadget_start()
2405 dev_err(dwc->dev, "failed to request irq #%d --> %d\n", in dwc3_gadget_start()
2410 spin_lock_irqsave(&dwc->lock, flags); in dwc3_gadget_start()
2411 if (dwc->gadget_driver) { in dwc3_gadget_start()
2412 dev_err(dwc->dev, "%s is already bound to %s\n", in dwc3_gadget_start()
2413 dwc->gadget->name, in dwc3_gadget_start()
2414 dwc->gadget_driver->driver.name); in dwc3_gadget_start()
2415 ret = -EBUSY; in dwc3_gadget_start()
2419 dwc->gadget_driver = driver; in dwc3_gadget_start()
2420 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_gadget_start()
2425 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_gadget_start()
2435 __dwc3_gadget_ep_disable(dwc->eps[0]); in __dwc3_gadget_stop()
2436 __dwc3_gadget_ep_disable(dwc->eps[1]); in __dwc3_gadget_stop()
2444 spin_lock_irqsave(&dwc->lock, flags); in dwc3_gadget_stop()
2445 dwc->gadget_driver = NULL; in dwc3_gadget_stop()
2446 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_gadget_stop()
2448 free_irq(dwc->irq_gadget, dwc->ev_buf); in dwc3_gadget_stop()
2458 params->besl_baseline = USB_DEFAULT_BESL_UNSPECIFIED; in dwc3_gadget_config_params()
2459 params->besl_deep = USB_DEFAULT_BESL_UNSPECIFIED; in dwc3_gadget_config_params()
2462 if (!dwc->dis_enblslpm_quirk) { in dwc3_gadget_config_params()
2472 params->besl_baseline = 1; in dwc3_gadget_config_params()
2473 if (dwc->is_utmi_l1_suspend) in dwc3_gadget_config_params()
2474 params->besl_deep = in dwc3_gadget_config_params()
2475 clamp_t(u8, dwc->hird_threshold, 2, 15); in dwc3_gadget_config_params()
2479 if (dwc->dis_u1_entry_quirk) in dwc3_gadget_config_params()
2480 params->bU1devExitLat = 0; in dwc3_gadget_config_params()
2482 params->bU1devExitLat = DWC3_DEFAULT_U1_DEV_EXIT_LAT; in dwc3_gadget_config_params()
2485 if (dwc->dis_u2_entry_quirk) in dwc3_gadget_config_params()
2486 params->bU2DevExitLat = 0; in dwc3_gadget_config_params()
2488 params->bU2DevExitLat = in dwc3_gadget_config_params()
2499 spin_lock_irqsave(&dwc->lock, flags); in dwc3_gadget_set_speed()
2500 reg = dwc3_readl(dwc->regs, DWC3_DCFG); in dwc3_gadget_set_speed()
2506 * bit if we try to force the IP to USB2-only mode. in dwc3_gadget_set_speed()
2517 !dwc->dis_metastability_quirk) { in dwc3_gadget_set_speed()
2540 dev_err(dwc->dev, "invalid speed (%d)\n", speed); in dwc3_gadget_set_speed()
2548 dwc3_writel(dwc->regs, DWC3_DCFG, reg); in dwc3_gadget_set_speed()
2550 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_gadget_set_speed()
2557 .pullup = dwc3_gadget_pullup,
2564 /* -------------------------------------------------------------------------- */
2568 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_init_control_endpoint()
2570 usb_ep_set_maxpacket_limit(&dep->endpoint, 512); in dwc3_gadget_init_control_endpoint()
2571 dep->endpoint.maxburst = 1; in dwc3_gadget_init_control_endpoint()
2572 dep->endpoint.ops = &dwc3_gadget_ep0_ops; in dwc3_gadget_init_control_endpoint()
2573 if (!dep->direction) in dwc3_gadget_init_control_endpoint()
2574 dwc->gadget->ep0 = &dep->endpoint; in dwc3_gadget_init_control_endpoint()
2576 dep->endpoint.caps.type_control = true; in dwc3_gadget_init_control_endpoint()
2583 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_init_in_endpoint()
2587 mdwidth = DWC3_MDWIDTH(dwc->hwparams.hwparams0); in dwc3_gadget_init_in_endpoint()
2589 mdwidth += DWC3_GHWPARAMS6_MDWIDTH(dwc->hwparams.hwparams6); in dwc3_gadget_init_in_endpoint()
2594 size = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(dep->number >> 1)); in dwc3_gadget_init_in_endpoint()
2610 if (dwc->maximum_speed >= USB_SPEED_SUPER) in dwc3_gadget_init_in_endpoint()
2615 usb_ep_set_maxpacket_limit(&dep->endpoint, size); in dwc3_gadget_init_in_endpoint()
2617 dep->endpoint.max_streams = 16; in dwc3_gadget_init_in_endpoint()
2618 dep->endpoint.ops = &dwc3_gadget_ep_ops; in dwc3_gadget_init_in_endpoint()
2619 list_add_tail(&dep->endpoint.ep_list, in dwc3_gadget_init_in_endpoint()
2620 &dwc->gadget->ep_list); in dwc3_gadget_init_in_endpoint()
2621 dep->endpoint.caps.type_iso = true; in dwc3_gadget_init_in_endpoint()
2622 dep->endpoint.caps.type_bulk = true; in dwc3_gadget_init_in_endpoint()
2623 dep->endpoint.caps.type_int = true; in dwc3_gadget_init_in_endpoint()
2630 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_init_out_endpoint()
2634 mdwidth = DWC3_MDWIDTH(dwc->hwparams.hwparams0); in dwc3_gadget_init_out_endpoint()
2636 mdwidth += DWC3_GHWPARAMS6_MDWIDTH(dwc->hwparams.hwparams6); in dwc3_gadget_init_out_endpoint()
2642 size = dwc3_readl(dwc->regs, DWC3_GRXFIFOSIZ(0)); in dwc3_gadget_init_out_endpoint()
2659 size -= (3 * 8) + 16; in dwc3_gadget_init_out_endpoint()
2665 usb_ep_set_maxpacket_limit(&dep->endpoint, size); in dwc3_gadget_init_out_endpoint()
2666 dep->endpoint.max_streams = 16; in dwc3_gadget_init_out_endpoint()
2667 dep->endpoint.ops = &dwc3_gadget_ep_ops; in dwc3_gadget_init_out_endpoint()
2668 list_add_tail(&dep->endpoint.ep_list, in dwc3_gadget_init_out_endpoint()
2669 &dwc->gadget->ep_list); in dwc3_gadget_init_out_endpoint()
2670 dep->endpoint.caps.type_iso = true; in dwc3_gadget_init_out_endpoint()
2671 dep->endpoint.caps.type_bulk = true; in dwc3_gadget_init_out_endpoint()
2672 dep->endpoint.caps.type_int = true; in dwc3_gadget_init_out_endpoint()
2686 return -ENOMEM; in dwc3_gadget_init_endpoint()
2688 dep->dwc = dwc; in dwc3_gadget_init_endpoint()
2689 dep->number = epnum; in dwc3_gadget_init_endpoint()
2690 dep->direction = direction; in dwc3_gadget_init_endpoint()
2691 dep->regs = dwc->regs + DWC3_DEP_BASE(epnum); in dwc3_gadget_init_endpoint()
2692 dwc->eps[epnum] = dep; in dwc3_gadget_init_endpoint()
2693 dep->combo_num = 0; in dwc3_gadget_init_endpoint()
2694 dep->start_cmd_status = 0; in dwc3_gadget_init_endpoint()
2696 snprintf(dep->name, sizeof(dep->name), "ep%u%s", num, in dwc3_gadget_init_endpoint()
2699 dep->endpoint.name = dep->name; in dwc3_gadget_init_endpoint()
2701 if (!(dep->number > 1)) { in dwc3_gadget_init_endpoint()
2702 dep->endpoint.desc = &dwc3_gadget_ep0_desc; in dwc3_gadget_init_endpoint()
2703 dep->endpoint.comp_desc = NULL; in dwc3_gadget_init_endpoint()
2716 dep->endpoint.caps.dir_in = direction; in dwc3_gadget_init_endpoint()
2717 dep->endpoint.caps.dir_out = !direction; in dwc3_gadget_init_endpoint()
2719 INIT_LIST_HEAD(&dep->pending_list); in dwc3_gadget_init_endpoint()
2720 INIT_LIST_HEAD(&dep->started_list); in dwc3_gadget_init_endpoint()
2721 INIT_LIST_HEAD(&dep->cancelled_list); in dwc3_gadget_init_endpoint()
2732 INIT_LIST_HEAD(&dwc->gadget->ep_list); in dwc3_gadget_init_endpoints()
2751 dep = dwc->eps[epnum]; in dwc3_gadget_free_endpoints()
2756 * bi-directional USB endpoint 0. in dwc3_gadget_free_endpoints()
2765 list_del(&dep->endpoint.ep_list); in dwc3_gadget_free_endpoints()
2768 debugfs_remove_recursive(debugfs_lookup(dep->name, dwc->root)); in dwc3_gadget_free_endpoints()
2773 /* -------------------------------------------------------------------------- */
2784 req->num_trbs--; in dwc3_gadget_ep_reclaim_completed_trb()
2796 if (chain && (trb->ctrl & DWC3_TRB_CTRL_HWO)) in dwc3_gadget_ep_reclaim_completed_trb()
2797 trb->ctrl &= ~DWC3_TRB_CTRL_HWO; in dwc3_gadget_ep_reclaim_completed_trb()
2801 * have the Isoc-First type. Track and report its interval frame number. in dwc3_gadget_ep_reclaim_completed_trb()
2803 if (usb_endpoint_xfer_isoc(dep->endpoint.desc) && in dwc3_gadget_ep_reclaim_completed_trb()
2804 (trb->ctrl & DWC3_TRBCTL_ISOCHRONOUS_FIRST)) { in dwc3_gadget_ep_reclaim_completed_trb()
2807 frame_number = DWC3_TRB_CTRL_GET_SID_SOFN(trb->ctrl); in dwc3_gadget_ep_reclaim_completed_trb()
2808 frame_number &= ~(dep->interval - 1); in dwc3_gadget_ep_reclaim_completed_trb()
2809 req->request.frame_number = frame_number; in dwc3_gadget_ep_reclaim_completed_trb()
2815 * TRB. Don't add it to req->remaining calculation. in dwc3_gadget_ep_reclaim_completed_trb()
2817 if (trb->bpl == lower_32_bits(dep->dwc->bounce_addr) && in dwc3_gadget_ep_reclaim_completed_trb()
2818 trb->bph == upper_32_bits(dep->dwc->bounce_addr)) { in dwc3_gadget_ep_reclaim_completed_trb()
2819 trb->ctrl &= ~DWC3_TRB_CTRL_HWO; in dwc3_gadget_ep_reclaim_completed_trb()
2823 count = trb->size & DWC3_TRB_SIZE_MASK; in dwc3_gadget_ep_reclaim_completed_trb()
2824 req->remaining += count; in dwc3_gadget_ep_reclaim_completed_trb()
2826 if ((trb->ctrl & DWC3_TRB_CTRL_HWO) && status != -ESHUTDOWN) in dwc3_gadget_ep_reclaim_completed_trb()
2829 if (event->status & DEPEVT_STATUS_SHORT && !chain) in dwc3_gadget_ep_reclaim_completed_trb()
2832 if ((trb->ctrl & DWC3_TRB_CTRL_ISP_IMI) && in dwc3_gadget_ep_reclaim_completed_trb()
2833 DWC3_TRB_SIZE_TRBSTS(trb->size) == DWC3_TRBSTS_MISSED_ISOC) in dwc3_gadget_ep_reclaim_completed_trb()
2836 if ((trb->ctrl & DWC3_TRB_CTRL_IOC) || in dwc3_gadget_ep_reclaim_completed_trb()
2837 (trb->ctrl & DWC3_TRB_CTRL_LST)) in dwc3_gadget_ep_reclaim_completed_trb()
2847 struct dwc3_trb *trb = &dep->trb_pool[dep->trb_dequeue]; in dwc3_gadget_ep_reclaim_trb_sg()
2848 struct scatterlist *sg = req->sg; in dwc3_gadget_ep_reclaim_trb_sg()
2850 unsigned int num_queued = req->num_queued_sgs; in dwc3_gadget_ep_reclaim_trb_sg()
2855 trb = &dep->trb_pool[dep->trb_dequeue]; in dwc3_gadget_ep_reclaim_trb_sg()
2857 req->sg = sg_next(s); in dwc3_gadget_ep_reclaim_trb_sg()
2858 req->num_queued_sgs--; in dwc3_gadget_ep_reclaim_trb_sg()
2873 struct dwc3_trb *trb = &dep->trb_pool[dep->trb_dequeue]; in dwc3_gadget_ep_reclaim_trb_linear()
2881 return req->num_pending_sgs == 0 && req->num_queued_sgs == 0; in dwc3_gadget_ep_request_completed()
2891 if (req->request.num_mapped_sgs) in dwc3_gadget_ep_cleanup_completed_request()
2898 req->request.actual = req->request.length - req->remaining; in dwc3_gadget_ep_cleanup_completed_request()
2903 if (req->needs_extra_trb) { in dwc3_gadget_ep_cleanup_completed_request()
2906 req->needs_extra_trb = false; in dwc3_gadget_ep_cleanup_completed_request()
2915 if (req->request.no_interrupt) { in dwc3_gadget_ep_cleanup_completed_request()
2918 trb = dwc3_ep_prev_trb(dep, dep->trb_dequeue); in dwc3_gadget_ep_cleanup_completed_request()
2919 switch (DWC3_TRB_SIZE_TRBSTS(trb->size)) { in dwc3_gadget_ep_cleanup_completed_request()
2922 request_status = -EXDEV; in dwc3_gadget_ep_cleanup_completed_request()
2949 list_for_each_entry_safe(req, tmp, &dep->started_list, list) { in dwc3_gadget_ep_cleanup_completed_requests()
2963 if (!list_empty(&dep->pending_list)) in dwc3_gadget_ep_should_continue()
2970 req = next_request(&dep->started_list); in dwc3_gadget_ep_should_continue()
2980 dep->frame_number = event->parameters; in dwc3_gadget_endpoint_frame_from_event()
2986 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_endpoint_trbs_complete()
2991 if (dep->flags & DWC3_EP_END_TRANSFER_PENDING) in dwc3_gadget_endpoint_trbs_complete()
2994 if (!dep->endpoint.desc) in dwc3_gadget_endpoint_trbs_complete()
2997 if (usb_endpoint_xfer_isoc(dep->endpoint.desc) && in dwc3_gadget_endpoint_trbs_complete()
2998 list_empty(&dep->started_list) && in dwc3_gadget_endpoint_trbs_complete()
2999 (list_empty(&dep->pending_list) || status == -EXDEV)) in dwc3_gadget_endpoint_trbs_complete()
3007 * WORKAROUND: This is the 2nd half of U1/U2 -> U0 workaround. in dwc3_gadget_endpoint_trbs_complete()
3015 dep = dwc->eps[i]; in dwc3_gadget_endpoint_trbs_complete()
3017 if (!(dep->flags & DWC3_EP_ENABLED)) in dwc3_gadget_endpoint_trbs_complete()
3020 if (!list_empty(&dep->started_list)) in dwc3_gadget_endpoint_trbs_complete()
3024 reg = dwc3_readl(dwc->regs, DWC3_DCTL); in dwc3_gadget_endpoint_trbs_complete()
3025 reg |= dwc->u1u2; in dwc3_gadget_endpoint_trbs_complete()
3026 dwc3_writel(dwc->regs, DWC3_DCTL, reg); in dwc3_gadget_endpoint_trbs_complete()
3028 dwc->u1u2 = 0; in dwc3_gadget_endpoint_trbs_complete()
3039 if (!dep->endpoint.desc) in dwc3_gadget_endpoint_transfer_in_progress()
3042 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) in dwc3_gadget_endpoint_transfer_in_progress()
3045 if (event->status & DEPEVT_STATUS_BUSERR) in dwc3_gadget_endpoint_transfer_in_progress()
3046 status = -ECONNRESET; in dwc3_gadget_endpoint_transfer_in_progress()
3048 if (event->status & DEPEVT_STATUS_MISSED_ISOC) in dwc3_gadget_endpoint_transfer_in_progress()
3049 status = -EXDEV; in dwc3_gadget_endpoint_transfer_in_progress()
3059 dep->flags &= ~DWC3_EP_TRANSFER_STARTED; in dwc3_gadget_endpoint_transfer_complete()
3061 if (event->status & DEPEVT_STATUS_BUSERR) in dwc3_gadget_endpoint_transfer_complete()
3062 status = -ECONNRESET; in dwc3_gadget_endpoint_transfer_complete()
3065 dep->flags &= ~DWC3_EP_WAIT_TRANSFER_COMPLETE; in dwc3_gadget_endpoint_transfer_complete()
3081 if (dep->flags & DWC3_EP_END_TRANSFER_PENDING) in dwc3_gadget_endpoint_transfer_not_ready()
3090 u8 cmd = DEPEVT_PARAMETER_CMD(event->parameters); in dwc3_gadget_endpoint_command_complete()
3100 if (dep->stream_capable) in dwc3_gadget_endpoint_command_complete()
3101 dep->flags |= DWC3_EP_IGNORE_NEXT_NOSTREAM; in dwc3_gadget_endpoint_command_complete()
3103 dep->flags &= ~DWC3_EP_END_TRANSFER_PENDING; in dwc3_gadget_endpoint_command_complete()
3104 dep->flags &= ~DWC3_EP_TRANSFER_STARTED; in dwc3_gadget_endpoint_command_complete()
3107 if (dep->flags & DWC3_EP_PENDING_CLEAR_STALL) { in dwc3_gadget_endpoint_command_complete()
3108 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_endpoint_command_complete()
3110 dep->flags &= ~DWC3_EP_PENDING_CLEAR_STALL; in dwc3_gadget_endpoint_command_complete()
3112 struct usb_ep *ep0 = &dwc->eps[0]->endpoint; in dwc3_gadget_endpoint_command_complete()
3114 dev_err(dwc->dev, "failed to clear STALL on %s\n", dep->name); in dwc3_gadget_endpoint_command_complete()
3115 if (dwc->delayed_status) in dwc3_gadget_endpoint_command_complete()
3120 dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE); in dwc3_gadget_endpoint_command_complete()
3121 if (dwc->delayed_status) in dwc3_gadget_endpoint_command_complete()
3125 if ((dep->flags & DWC3_EP_DELAY_START) && in dwc3_gadget_endpoint_command_complete()
3126 !usb_endpoint_xfer_isoc(dep->endpoint.desc)) in dwc3_gadget_endpoint_command_complete()
3129 dep->flags &= ~DWC3_EP_DELAY_START; in dwc3_gadget_endpoint_command_complete()
3135 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_endpoint_stream_event()
3137 if (event->status == DEPEVT_STREAMEVT_FOUND) { in dwc3_gadget_endpoint_stream_event()
3138 dep->flags |= DWC3_EP_FIRST_STREAM_PRIMED; in dwc3_gadget_endpoint_stream_event()
3143 switch (event->parameters) { in dwc3_gadget_endpoint_stream_event()
3152 if (dep->flags & DWC3_EP_FORCE_RESTART_STREAM) { in dwc3_gadget_endpoint_stream_event()
3153 if (dep->flags & DWC3_EP_FIRST_STREAM_PRIMED) in dwc3_gadget_endpoint_stream_event()
3154 dep->flags &= ~DWC3_EP_FORCE_RESTART_STREAM; in dwc3_gadget_endpoint_stream_event()
3156 dep->flags |= DWC3_EP_FIRST_STREAM_PRIMED; in dwc3_gadget_endpoint_stream_event()
3161 if ((dep->flags & DWC3_EP_IGNORE_NEXT_NOSTREAM) || in dwc3_gadget_endpoint_stream_event()
3162 !(dep->flags & DWC3_EP_FORCE_RESTART_STREAM) || in dwc3_gadget_endpoint_stream_event()
3163 !(dep->flags & DWC3_EP_WAIT_TRANSFER_COMPLETE)) in dwc3_gadget_endpoint_stream_event()
3183 dwc3_send_gadget_generic_command(dwc, cmd, dep->number); in dwc3_gadget_endpoint_stream_event()
3185 dep->flags |= DWC3_EP_DELAY_START; in dwc3_gadget_endpoint_stream_event()
3193 dep->flags &= ~DWC3_EP_IGNORE_NEXT_NOSTREAM; in dwc3_gadget_endpoint_stream_event()
3200 u8 epnum = event->endpoint_number; in dwc3_endpoint_interrupt()
3202 dep = dwc->eps[epnum]; in dwc3_endpoint_interrupt()
3204 if (!(dep->flags & DWC3_EP_ENABLED)) { in dwc3_endpoint_interrupt()
3205 if (!(dep->flags & DWC3_EP_TRANSFER_STARTED)) in dwc3_endpoint_interrupt()
3209 if (event->endpoint_event != DWC3_DEPEVT_EPCMDCMPLT) in dwc3_endpoint_interrupt()
3218 switch (event->endpoint_event) { in dwc3_endpoint_interrupt()
3241 if (dwc->gadget_driver && dwc->gadget_driver->disconnect) { in dwc3_disconnect_gadget()
3242 spin_unlock(&dwc->lock); in dwc3_disconnect_gadget()
3243 dwc->gadget_driver->disconnect(dwc->gadget); in dwc3_disconnect_gadget()
3244 spin_lock(&dwc->lock); in dwc3_disconnect_gadget()
3250 if (dwc->gadget_driver && dwc->gadget_driver->suspend) { in dwc3_suspend_gadget()
3251 spin_unlock(&dwc->lock); in dwc3_suspend_gadget()
3252 dwc->gadget_driver->suspend(dwc->gadget); in dwc3_suspend_gadget()
3253 spin_lock(&dwc->lock); in dwc3_suspend_gadget()
3259 if (dwc->gadget_driver && dwc->gadget_driver->resume) { in dwc3_resume_gadget()
3260 spin_unlock(&dwc->lock); in dwc3_resume_gadget()
3261 dwc->gadget_driver->resume(dwc->gadget); in dwc3_resume_gadget()
3262 spin_lock(&dwc->lock); in dwc3_resume_gadget()
3268 if (!dwc->gadget_driver) in dwc3_reset_gadget()
3271 if (dwc->gadget->speed != USB_SPEED_UNKNOWN) { in dwc3_reset_gadget()
3272 spin_unlock(&dwc->lock); in dwc3_reset_gadget()
3273 usb_gadget_udc_reset(dwc->gadget, dwc->gadget_driver); in dwc3_reset_gadget()
3274 spin_lock(&dwc->lock); in dwc3_reset_gadget()
3281 if (!(dep->flags & DWC3_EP_TRANSFER_STARTED) || in dwc3_stop_active_transfer()
3282 (dep->flags & DWC3_EP_END_TRANSFER_PENDING)) in dwc3_stop_active_transfer()
3327 dep = dwc->eps[epnum]; in dwc3_clear_stall_all_ep()
3331 if (!(dep->flags & DWC3_EP_STALL)) in dwc3_clear_stall_all_ep()
3334 dep->flags &= ~DWC3_EP_STALL; in dwc3_clear_stall_all_ep()
3347 reg = dwc3_readl(dwc->regs, DWC3_DCTL); in dwc3_gadget_disconnect_interrupt()
3354 dwc->gadget->speed = USB_SPEED_UNKNOWN; in dwc3_gadget_disconnect_interrupt()
3355 dwc->setup_packet_pending = false; in dwc3_gadget_disconnect_interrupt()
3356 usb_gadget_set_state(dwc->gadget, USB_STATE_NOTATTACHED); in dwc3_gadget_disconnect_interrupt()
3358 dwc->connected = false; in dwc3_gadget_disconnect_interrupt()
3367 * drivers to stop any active transfers through ep disable. in dwc3_gadget_reset_interrupt()
3368 * However, for functions which defer ep disable, such as mass in dwc3_gadget_reset_interrupt()
3372 dwc->connected = false; in dwc3_gadget_reset_interrupt()
3401 if (dwc->setup_packet_pending) in dwc3_gadget_reset_interrupt()
3408 * Section 4.1.2 Table 4-2, it states that during a USB reset, the SW in dwc3_gadget_reset_interrupt()
3413 dwc->connected = true; in dwc3_gadget_reset_interrupt()
3415 reg = dwc3_readl(dwc->regs, DWC3_DCTL); in dwc3_gadget_reset_interrupt()
3418 dwc->test_mode = false; in dwc3_gadget_reset_interrupt()
3422 reg = dwc3_readl(dwc->regs, DWC3_DCFG); in dwc3_gadget_reset_interrupt()
3424 dwc3_writel(dwc->regs, DWC3_DCFG, reg); in dwc3_gadget_reset_interrupt()
3434 reg = dwc3_readl(dwc->regs, DWC3_DSTS); in dwc3_gadget_conndone_interrupt()
3436 dwc->speed = speed; in dwc3_gadget_conndone_interrupt()
3450 dwc->gadget->ep0->maxpacket = 512; in dwc3_gadget_conndone_interrupt()
3451 dwc->gadget->speed = USB_SPEED_SUPER_PLUS; in dwc3_gadget_conndone_interrupt()
3471 dwc->gadget->ep0->maxpacket = 512; in dwc3_gadget_conndone_interrupt()
3472 dwc->gadget->speed = USB_SPEED_SUPER; in dwc3_gadget_conndone_interrupt()
3476 dwc->gadget->ep0->maxpacket = 64; in dwc3_gadget_conndone_interrupt()
3477 dwc->gadget->speed = USB_SPEED_HIGH; in dwc3_gadget_conndone_interrupt()
3481 dwc->gadget->ep0->maxpacket = 64; in dwc3_gadget_conndone_interrupt()
3482 dwc->gadget->speed = USB_SPEED_FULL; in dwc3_gadget_conndone_interrupt()
3486 dwc->gadget->ep0->maxpacket = 8; in dwc3_gadget_conndone_interrupt()
3487 dwc->gadget->speed = USB_SPEED_LOW; in dwc3_gadget_conndone_interrupt()
3491 dwc->eps[1]->endpoint.maxpacket = dwc->gadget->ep0->maxpacket; in dwc3_gadget_conndone_interrupt()
3496 !dwc->usb2_gadget_lpm_disable && in dwc3_gadget_conndone_interrupt()
3499 reg = dwc3_readl(dwc->regs, DWC3_DCFG); in dwc3_gadget_conndone_interrupt()
3501 dwc3_writel(dwc->regs, DWC3_DCFG, reg); in dwc3_gadget_conndone_interrupt()
3503 reg = dwc3_readl(dwc->regs, DWC3_DCTL); in dwc3_gadget_conndone_interrupt()
3506 reg |= DWC3_DCTL_HIRD_THRES(dwc->hird_threshold | in dwc3_gadget_conndone_interrupt()
3507 (dwc->is_utmi_l1_suspend << 4)); in dwc3_gadget_conndone_interrupt()
3515 WARN_ONCE(DWC3_VER_IS_PRIOR(DWC3, 240A) && dwc->has_lpm_erratum, in dwc3_gadget_conndone_interrupt()
3518 if (dwc->has_lpm_erratum && !DWC3_VER_IS_PRIOR(DWC3, 240A)) in dwc3_gadget_conndone_interrupt()
3519 reg |= DWC3_DCTL_NYET_THRES(dwc->lpm_nyet_threshold); in dwc3_gadget_conndone_interrupt()
3523 if (dwc->usb2_gadget_lpm_disable) { in dwc3_gadget_conndone_interrupt()
3524 reg = dwc3_readl(dwc->regs, DWC3_DCFG); in dwc3_gadget_conndone_interrupt()
3526 dwc3_writel(dwc->regs, DWC3_DCFG, reg); in dwc3_gadget_conndone_interrupt()
3529 reg = dwc3_readl(dwc->regs, DWC3_DCTL); in dwc3_gadget_conndone_interrupt()
3534 dep = dwc->eps[0]; in dwc3_gadget_conndone_interrupt()
3537 dev_err(dwc->dev, "failed to enable %s\n", dep->name); in dwc3_gadget_conndone_interrupt()
3541 dep = dwc->eps[1]; in dwc3_gadget_conndone_interrupt()
3544 dev_err(dwc->dev, "failed to enable %s\n", dep->name); in dwc3_gadget_conndone_interrupt()
3564 if (dwc->gadget_driver && dwc->gadget_driver->resume) { in dwc3_gadget_wakeup_interrupt()
3565 spin_unlock(&dwc->lock); in dwc3_gadget_wakeup_interrupt()
3566 dwc->gadget_driver->resume(dwc->gadget); in dwc3_gadget_wakeup_interrupt()
3567 spin_lock(&dwc->lock); in dwc3_gadget_wakeup_interrupt()
3580 * host-initiated U3 exit. in dwc3_gadget_linksts_change_interrupt()
3591 * STAR#9000570034 RTL: SS Resume event generated in non-Hibernation in dwc3_gadget_linksts_change_interrupt()
3594 pwropt = DWC3_GHWPARAMS1_EN_PWROPT(dwc->hwparams.hwparams1); in dwc3_gadget_linksts_change_interrupt()
3597 if ((dwc->link_state == DWC3_LINK_STATE_U3) && in dwc3_gadget_linksts_change_interrupt()
3609 * suggested workaround is to disable DCTL[12:9] bits if we're in dwc3_gadget_linksts_change_interrupt()
3618 * STAR#9000446952: RTL: Device SS : if U1/U2 ->U0 takes >128us in dwc3_gadget_linksts_change_interrupt()
3626 switch (dwc->link_state) { in dwc3_gadget_linksts_change_interrupt()
3629 reg = dwc3_readl(dwc->regs, DWC3_DCTL); in dwc3_gadget_linksts_change_interrupt()
3635 if (!dwc->u1u2) in dwc3_gadget_linksts_change_interrupt()
3636 dwc->u1u2 = reg & u1u2; in dwc3_gadget_linksts_change_interrupt()
3651 if (dwc->speed == USB_SPEED_SUPER) in dwc3_gadget_linksts_change_interrupt()
3666 dwc->link_state = next; in dwc3_gadget_linksts_change_interrupt()
3674 if (dwc->link_state != next && next == DWC3_LINK_STATE_U3) in dwc3_gadget_suspend_interrupt()
3677 dwc->link_state = next; in dwc3_gadget_suspend_interrupt()
3698 if (is_ss ^ (dwc->speed == USB_SPEED_SUPER)) in dwc3_gadget_hibernation_interrupt()
3707 switch (event->type) { in dwc3_gadget_interrupt()
3721 if (dev_WARN_ONCE(dwc->dev, !dwc->has_hibernation, in dwc3_gadget_interrupt()
3725 dwc3_gadget_hibernation_interrupt(dwc, event->event_info); in dwc3_gadget_interrupt()
3728 dwc3_gadget_linksts_change_interrupt(dwc, event->event_info); in dwc3_gadget_interrupt()
3737 if (dwc->gadget->state >= USB_STATE_CONFIGURED) in dwc3_gadget_interrupt()
3739 event->event_info); in dwc3_gadget_interrupt()
3748 dev_WARN(dwc->dev, "UNKNOWN IRQ %d\n", event->type); in dwc3_gadget_interrupt()
3755 trace_dwc3_event(event->raw, dwc); in dwc3_process_event_entry()
3757 if (!event->type.is_devspec) in dwc3_process_event_entry()
3758 dwc3_endpoint_interrupt(dwc, &event->depevt); in dwc3_process_event_entry()
3759 else if (event->type.type == DWC3_EVENT_TYPE_DEV) in dwc3_process_event_entry()
3760 dwc3_gadget_interrupt(dwc, &event->devt); in dwc3_process_event_entry()
3762 dev_err(dwc->dev, "UNKNOWN IRQ type %d\n", event->raw); in dwc3_process_event_entry()
3767 struct dwc3 *dwc = evt->dwc; in dwc3_process_event_buf()
3772 left = evt->count; in dwc3_process_event_buf()
3774 if (!(evt->flags & DWC3_EVENT_PENDING)) in dwc3_process_event_buf()
3780 event.raw = *(u32 *) (evt->cache + evt->lpos); in dwc3_process_event_buf()
3793 evt->lpos = (evt->lpos + 4) % evt->length; in dwc3_process_event_buf()
3794 left -= 4; in dwc3_process_event_buf()
3797 evt->count = 0; in dwc3_process_event_buf()
3801 reg = dwc3_readl(dwc->regs, DWC3_GEVNTSIZ(0)); in dwc3_process_event_buf()
3803 dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0), reg); in dwc3_process_event_buf()
3805 if (dwc->imod_interval) { in dwc3_process_event_buf()
3806 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), DWC3_GEVNTCOUNT_EHB); in dwc3_process_event_buf()
3807 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), dwc->imod_interval); in dwc3_process_event_buf()
3811 evt->flags &= ~DWC3_EVENT_PENDING; in dwc3_process_event_buf()
3819 struct dwc3 *dwc = evt->dwc; in dwc3_thread_interrupt()
3824 spin_lock_irqsave(&dwc->lock, flags); in dwc3_thread_interrupt()
3826 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_thread_interrupt()
3834 struct dwc3 *dwc = evt->dwc; in dwc3_check_event_buf()
3839 if (pm_runtime_suspended(dwc->dev)) { in dwc3_check_event_buf()
3840 dwc->pending_events = true; in dwc3_check_event_buf()
3846 pm_runtime_get(dwc->dev); in dwc3_check_event_buf()
3847 disable_irq_nosync(dwc->irq_gadget); in dwc3_check_event_buf()
3852 * With PCIe legacy interrupt, test shows that top-half irq handler can in dwc3_check_event_buf()
3853 * be called again after HW interrupt deassertion. Check if bottom-half in dwc3_check_event_buf()
3857 if (evt->flags & DWC3_EVENT_PENDING) in dwc3_check_event_buf()
3860 count = dwc3_readl(dwc->regs, DWC3_GEVNTCOUNT(0)); in dwc3_check_event_buf()
3865 if (count > evt->length) { in dwc3_check_event_buf()
3866 dev_err_ratelimited(dwc->dev, "invalid count(%u) > evt->length(%u)\n", in dwc3_check_event_buf()
3867 count, evt->length); in dwc3_check_event_buf()
3871 evt->count = count; in dwc3_check_event_buf()
3872 evt->flags |= DWC3_EVENT_PENDING; in dwc3_check_event_buf()
3875 reg = dwc3_readl(dwc->regs, DWC3_GEVNTSIZ(0)); in dwc3_check_event_buf()
3877 dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0), reg); in dwc3_check_event_buf()
3879 amount = min(count, evt->length - evt->lpos); in dwc3_check_event_buf()
3880 memcpy(evt->cache + evt->lpos, evt->buf + evt->lpos, amount); in dwc3_check_event_buf()
3883 memcpy(evt->cache, evt->buf, count - amount); in dwc3_check_event_buf()
3885 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), count); in dwc3_check_event_buf()
3899 struct platform_device *dwc3_pdev = to_platform_device(dwc->dev); in dwc3_gadget_get_irq()
3906 if (irq == -EPROBE_DEFER) in dwc3_gadget_get_irq()
3913 if (irq == -EPROBE_DEFER) in dwc3_gadget_get_irq()
3921 irq = -EINVAL; in dwc3_gadget_get_irq()
3935 * dwc3_gadget_init - initializes gadget related registers
3952 dwc->irq_gadget = irq; in dwc3_gadget_init()
3954 dwc->ep0_trb = dma_alloc_coherent(dwc->sysdev, in dwc3_gadget_init()
3955 sizeof(*dwc->ep0_trb) * 2, in dwc3_gadget_init()
3956 &dwc->ep0_trb_addr, GFP_KERNEL); in dwc3_gadget_init()
3957 if (!dwc->ep0_trb) { in dwc3_gadget_init()
3958 dev_err(dwc->dev, "failed to allocate ep0 trb\n"); in dwc3_gadget_init()
3959 ret = -ENOMEM; in dwc3_gadget_init()
3963 dwc->setup_buf = kzalloc(DWC3_EP0_SETUP_SIZE, GFP_KERNEL); in dwc3_gadget_init()
3964 if (!dwc->setup_buf) { in dwc3_gadget_init()
3965 ret = -ENOMEM; in dwc3_gadget_init()
3969 dwc->bounce = dma_alloc_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE, in dwc3_gadget_init()
3970 &dwc->bounce_addr, GFP_KERNEL); in dwc3_gadget_init()
3971 if (!dwc->bounce) { in dwc3_gadget_init()
3972 ret = -ENOMEM; in dwc3_gadget_init()
3976 init_completion(&dwc->ep0_in_setup); in dwc3_gadget_init()
3977 dwc->gadget = kzalloc(sizeof(struct usb_gadget), GFP_KERNEL); in dwc3_gadget_init()
3978 if (!dwc->gadget) { in dwc3_gadget_init()
3979 ret = -ENOMEM; in dwc3_gadget_init()
3984 usb_initialize_gadget(dwc->dev, dwc->gadget, dwc_gadget_release); in dwc3_gadget_init()
3985 dev = &dwc->gadget->dev; in dwc3_gadget_init()
3986 dev->platform_data = dwc; in dwc3_gadget_init()
3987 dwc->gadget->ops = &dwc3_gadget_ops; in dwc3_gadget_init()
3988 dwc->gadget->speed = USB_SPEED_UNKNOWN; in dwc3_gadget_init()
3989 dwc->gadget->sg_supported = true; in dwc3_gadget_init()
3990 dwc->gadget->name = "dwc3-gadget"; in dwc3_gadget_init()
3991 dwc->gadget->lpm_capable = !dwc->usb2_gadget_lpm_disable; in dwc3_gadget_init()
4010 !dwc->dis_metastability_quirk) in dwc3_gadget_init()
4011 dev_info(dwc->dev, "changing max_speed on rev %08x\n", in dwc3_gadget_init()
4012 dwc->revision); in dwc3_gadget_init()
4014 dwc->gadget->max_speed = dwc->maximum_speed; in dwc3_gadget_init()
4021 ret = dwc3_gadget_init_endpoints(dwc, dwc->num_eps); in dwc3_gadget_init()
4025 ret = usb_add_gadget(dwc->gadget); in dwc3_gadget_init()
4027 dev_err(dwc->dev, "failed to add gadget\n"); in dwc3_gadget_init()
4031 dwc3_gadget_set_speed(dwc->gadget, dwc->maximum_speed); in dwc3_gadget_init()
4038 usb_put_gadget(dwc->gadget); in dwc3_gadget_init()
4039 dwc->gadget = NULL; in dwc3_gadget_init()
4041 dma_free_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE, dwc->bounce, in dwc3_gadget_init()
4042 dwc->bounce_addr); in dwc3_gadget_init()
4045 kfree(dwc->setup_buf); in dwc3_gadget_init()
4048 dma_free_coherent(dwc->sysdev, sizeof(*dwc->ep0_trb) * 2, in dwc3_gadget_init()
4049 dwc->ep0_trb, dwc->ep0_trb_addr); in dwc3_gadget_init()
4055 /* -------------------------------------------------------------------------- */
4059 if (!dwc->gadget) in dwc3_gadget_exit()
4062 usb_del_gadget(dwc->gadget); in dwc3_gadget_exit()
4064 usb_put_gadget(dwc->gadget); in dwc3_gadget_exit()
4065 dma_free_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE, dwc->bounce, in dwc3_gadget_exit()
4066 dwc->bounce_addr); in dwc3_gadget_exit()
4067 kfree(dwc->setup_buf); in dwc3_gadget_exit()
4068 dma_free_coherent(dwc->sysdev, sizeof(*dwc->ep0_trb) * 2, in dwc3_gadget_exit()
4069 dwc->ep0_trb, dwc->ep0_trb_addr); in dwc3_gadget_exit()
4074 if (!dwc->gadget_driver) in dwc3_gadget_suspend()
4088 if (!dwc->gadget_driver || !dwc->softconnect) in dwc3_gadget_resume()
4110 if (dwc->pending_events) { in dwc3_gadget_process_pending_events()
4111 dwc3_interrupt(dwc->irq_gadget, dwc->ev_buf); in dwc3_gadget_process_pending_events()
4112 dwc3_thread_interrupt(dwc->irq_gadget, dwc->ev_buf); in dwc3_gadget_process_pending_events()
4113 pm_runtime_put(dwc->dev); in dwc3_gadget_process_pending_events()
4114 dwc->pending_events = false; in dwc3_gadget_process_pending_events()
4115 enable_irq(dwc->irq_gadget); in dwc3_gadget_process_pending_events()