Lines Matching +full:ring +full:- +full:disable +full:- +full:pullup
1 // SPDX-License-Identifier: GPL-2.0
3 * Cadence USBSS DRD Driver - gadget side.
5 * Copyright (C) 2018-2019 Cadence Design Systems.
6 * Copyright (C) 2017-2018 NXP
32 * Controller for OUT endpoints has shared on-chip buffers for all incoming
37 * Additionally the packets directed to one endpoint can block entire on-chip
59 #include <linux/dma-mapping.h>
65 #include "gadget-export.h"
81 * cdns3_clear_register_bit - clear bit in given register.
92 * cdns3_set_register_bit - set bit in given register.
103 * cdns3_ep_addr_to_index - Macro converts endpoint address to
118 dma_index = readl(&priv_dev->regs->ep_traddr) - priv_ep->trb_pool_dma; in cdns3_get_dma_pos()
124 * cdns3_next_request - returns next request from list
135 * cdns3_next_align_buf - returns next buffer from list
146 * cdns3_next_priv_request - returns next request from list
157 * select_ep - selects endpoint
163 if (priv_dev->selected_ep == ep) in cdns3_select_ep()
166 priv_dev->selected_ep = ep; in cdns3_select_ep()
167 writel(ep, &priv_dev->regs->ep_sel); in cdns3_select_ep()
171 * cdns3_get_tdl - gets current tdl for selected endpoint.
179 if (priv_dev->dev_ver < DEV_VER_V3) in cdns3_get_tdl()
180 return EP_CMD_TDL_GET(readl(&priv_dev->regs->ep_cmd)); in cdns3_get_tdl()
182 return readl(&priv_dev->regs->ep_tdl); in cdns3_get_tdl()
188 u32 offset = (char *)trb - (char *)priv_ep->trb_pool; in cdns3_trb_virt_to_dma()
190 return priv_ep->trb_pool_dma + offset; in cdns3_trb_virt_to_dma()
195 switch (priv_ep->type) { in cdns3_ring_size()
201 if (priv_ep->use_streams) in cdns3_ring_size()
210 struct cdns3_device *priv_dev = priv_ep->cdns3_dev; in cdns3_free_trb_pool()
212 if (priv_ep->trb_pool) { in cdns3_free_trb_pool()
213 dma_free_coherent(priv_dev->sysdev, in cdns3_free_trb_pool()
215 priv_ep->trb_pool, priv_ep->trb_pool_dma); in cdns3_free_trb_pool()
216 priv_ep->trb_pool = NULL; in cdns3_free_trb_pool()
221 * cdns3_allocate_trb_pool - Allocates TRB's pool for selected endpoint
224 * Function will return 0 on success or -ENOMEM on allocation error
228 struct cdns3_device *priv_dev = priv_ep->cdns3_dev; in cdns3_allocate_trb_pool()
233 if (priv_ep->trb_pool && priv_ep->alloc_ring_size < ring_size) in cdns3_allocate_trb_pool()
236 if (!priv_ep->trb_pool) { in cdns3_allocate_trb_pool()
237 priv_ep->trb_pool = dma_alloc_coherent(priv_dev->sysdev, in cdns3_allocate_trb_pool()
239 &priv_ep->trb_pool_dma, in cdns3_allocate_trb_pool()
241 if (!priv_ep->trb_pool) in cdns3_allocate_trb_pool()
242 return -ENOMEM; in cdns3_allocate_trb_pool()
244 priv_ep->alloc_ring_size = ring_size; in cdns3_allocate_trb_pool()
247 memset(priv_ep->trb_pool, 0, ring_size); in cdns3_allocate_trb_pool()
249 priv_ep->num_trbs = num_trbs; in cdns3_allocate_trb_pool()
251 if (!priv_ep->num) in cdns3_allocate_trb_pool()
255 link_trb = (priv_ep->trb_pool + (priv_ep->num_trbs - 1)); in cdns3_allocate_trb_pool()
257 if (priv_ep->use_streams) { in cdns3_allocate_trb_pool()
262 link_trb->control = 0; in cdns3_allocate_trb_pool()
264 link_trb->buffer = cpu_to_le32(TRB_BUFFER(priv_ep->trb_pool_dma)); in cdns3_allocate_trb_pool()
265 link_trb->control = cpu_to_le32(TRB_CYCLE | TRB_TYPE(TRB_LINK) | TRB_TOGGLE); in cdns3_allocate_trb_pool()
271 * cdns3_ep_stall_flush - Stalls and flushes selected endpoint
278 struct cdns3_device *priv_dev = priv_ep->cdns3_dev; in cdns3_ep_stall_flush()
284 &priv_dev->regs->ep_cmd); in cdns3_ep_stall_flush()
287 readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val, in cdns3_ep_stall_flush()
289 priv_ep->flags |= EP_STALLED; in cdns3_ep_stall_flush()
290 priv_ep->flags &= ~EP_STALL_PENDING; in cdns3_ep_stall_flush()
294 * cdns3_hw_reset_eps_config - reset endpoints configuration kept by controller.
301 writel(USB_CONF_CFGRST, &priv_dev->regs->usb_conf); in cdns3_hw_reset_eps_config()
304 priv_dev->hw_configured_flag = 0; in cdns3_hw_reset_eps_config()
305 priv_dev->onchip_used_size = 0; in cdns3_hw_reset_eps_config()
306 priv_dev->out_mem_is_allocated = 0; in cdns3_hw_reset_eps_config()
307 priv_dev->wait_for_setup = 0; in cdns3_hw_reset_eps_config()
308 priv_dev->using_streams = 0; in cdns3_hw_reset_eps_config()
311 if (priv_dev->eps[i]) in cdns3_hw_reset_eps_config()
312 priv_dev->eps[i]->flags &= ~EP_CONFIGURED; in cdns3_hw_reset_eps_config()
316 * cdns3_ep_inc_trb - increment a trb index.
329 if (*index == (trb_in_seg - 1)) { in cdns3_ep_inc_trb()
336 * cdns3_ep_inc_enq - increment endpoint's enqueue pointer
341 priv_ep->free_trbs--; in cdns3_ep_inc_enq()
342 cdns3_ep_inc_trb(&priv_ep->enqueue, &priv_ep->pcs, priv_ep->num_trbs); in cdns3_ep_inc_enq()
346 * cdns3_ep_inc_deq - increment endpoint's dequeue pointer
351 priv_ep->free_trbs++; in cdns3_ep_inc_deq()
352 cdns3_ep_inc_trb(&priv_ep->dequeue, &priv_ep->ccs, priv_ep->num_trbs); in cdns3_ep_inc_deq()
356 * cdns3_allow_enable_l1 - enable/disable permits to transition to L1.
358 * @enable: Enable/disable permit to transition to L1.
368 writel(USB_CONF_L1EN, &priv_dev->regs->usb_conf); in cdns3_allow_enable_l1()
370 writel(USB_CONF_L1DS, &priv_dev->regs->usb_conf); in cdns3_allow_enable_l1()
377 reg = readl(&priv_dev->regs->usb_sts); in cdns3_get_speed()
391 * cdns3_start_all_request - add to ring all request not started
395 * Returns return ENOMEM if transfer ring i not enough TRBs to start
403 u8 pending_empty = list_empty(&priv_ep->pending_req_list); in cdns3_start_all_request()
413 request = cdns3_next_request(&priv_ep->pending_req_list); in cdns3_start_all_request()
415 if ((priv_req->flags & REQUEST_INTERNAL) || in cdns3_start_all_request()
416 (priv_ep->flags & EP_TDLCHK_EN) || in cdns3_start_all_request()
417 priv_ep->use_streams) { in cdns3_start_all_request()
418 dev_dbg(priv_dev->dev, "Blocking external request\n"); in cdns3_start_all_request()
423 while (!list_empty(&priv_ep->deferred_req_list)) { in cdns3_start_all_request()
424 request = cdns3_next_request(&priv_ep->deferred_req_list); in cdns3_start_all_request()
426 if (!priv_ep->use_streams) { in cdns3_start_all_request()
429 priv_ep->stream_sg_idx = 0; in cdns3_start_all_request()
435 list_del(&request->list); in cdns3_start_all_request()
436 list_add_tail(&request->list, in cdns3_start_all_request()
437 &priv_ep->pending_req_list); in cdns3_start_all_request()
438 if (request->stream_id != 0 || (priv_ep->flags & EP_TDLCHK_EN)) in cdns3_start_all_request()
442 priv_ep->flags &= ~EP_RING_FULL; in cdns3_start_all_request()
449 * buffer for unblocking on-chip FIFO buffer. This flag will be cleared
453 if (!priv_ep->dir && priv_ep->type != USB_ENDPOINT_XFER_ISOC) { \
454 priv_ep->flags |= EP_QUIRK_EXTRA_BUF_DET; \
461 int length = request->actual + descmiss_req->actual; in __cdns3_descmiss_copy_data()
462 struct scatterlist *s = request->sg; in __cdns3_descmiss_copy_data()
465 if (length <= request->length) { in __cdns3_descmiss_copy_data()
466 memcpy(&((u8 *)request->buf)[request->actual], in __cdns3_descmiss_copy_data()
467 descmiss_req->buf, in __cdns3_descmiss_copy_data()
468 descmiss_req->actual); in __cdns3_descmiss_copy_data()
469 request->actual = length; in __cdns3_descmiss_copy_data()
472 request->status = -ENOMEM; in __cdns3_descmiss_copy_data()
478 memcpy(&((u8 *)p)[request->actual], in __cdns3_descmiss_copy_data()
479 descmiss_req->buf, in __cdns3_descmiss_copy_data()
480 descmiss_req->actual); in __cdns3_descmiss_copy_data()
481 request->actual = length; in __cdns3_descmiss_copy_data()
483 request->status = -ENOMEM; in __cdns3_descmiss_copy_data()
500 while (!list_empty(&priv_ep->wa2_descmiss_req_list)) { in cdns3_wa2_descmiss_copy_data()
504 cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list); in cdns3_wa2_descmiss_copy_data()
505 descmiss_req = &descmiss_priv_req->request; in cdns3_wa2_descmiss_copy_data()
508 if (descmiss_priv_req->flags & REQUEST_PENDING) in cdns3_wa2_descmiss_copy_data()
511 chunk_end = descmiss_priv_req->flags & REQUEST_INTERNAL_CH; in cdns3_wa2_descmiss_copy_data()
512 request->status = descmiss_req->status; in cdns3_wa2_descmiss_copy_data()
514 list_del_init(&descmiss_priv_req->list); in cdns3_wa2_descmiss_copy_data()
515 kfree(descmiss_req->buf); in cdns3_wa2_descmiss_copy_data()
516 cdns3_gadget_ep_free_request(&priv_ep->endpoint, descmiss_req); in cdns3_wa2_descmiss_copy_data()
517 --priv_ep->wa2_counter; in cdns3_wa2_descmiss_copy_data()
528 if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN && in cdns3_wa2_gadget_giveback()
529 priv_req->flags & REQUEST_INTERNAL) { in cdns3_wa2_gadget_giveback()
532 req = cdns3_next_request(&priv_ep->deferred_req_list); in cdns3_wa2_gadget_giveback()
534 priv_ep->descmis_req = NULL; in cdns3_wa2_gadget_giveback()
540 usb_gadget_unmap_request_by_dev(priv_dev->sysdev, req, in cdns3_wa2_gadget_giveback()
541 priv_ep->dir); in cdns3_wa2_gadget_giveback()
544 if (!(priv_ep->flags & EP_QUIRK_END_TRANSFER) && in cdns3_wa2_gadget_giveback()
545 req->length != req->actual) { in cdns3_wa2_gadget_giveback()
547 /* re-map the gadget request buffer*/ in cdns3_wa2_gadget_giveback()
548 usb_gadget_map_request_by_dev(priv_dev->sysdev, req, in cdns3_wa2_gadget_giveback()
549 usb_endpoint_dir_in(priv_ep->endpoint.desc)); in cdns3_wa2_gadget_giveback()
553 if (req->status == -EINPROGRESS) in cdns3_wa2_gadget_giveback()
554 req->status = 0; in cdns3_wa2_gadget_giveback()
556 list_del_init(&req->list); in cdns3_wa2_gadget_giveback()
561 return &priv_req->request; in cdns3_wa2_gadget_giveback()
572 * can disable handling of DESCMISS interrupt. Driver assumes that it in cdns3_wa2_gadget_ep_queue()
573 * can disable special treatment for this endpoint. in cdns3_wa2_gadget_ep_queue()
575 if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_DET) { in cdns3_wa2_gadget_ep_queue()
578 cdns3_select_ep(priv_dev, priv_ep->num | priv_ep->dir); in cdns3_wa2_gadget_ep_queue()
579 priv_ep->flags &= ~EP_QUIRK_EXTRA_BUF_DET; in cdns3_wa2_gadget_ep_queue()
580 reg = readl(&priv_dev->regs->ep_sts_en); in cdns3_wa2_gadget_ep_queue()
583 writel(reg, &priv_dev->regs->ep_sts_en); in cdns3_wa2_gadget_ep_queue()
586 if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN) { in cdns3_wa2_gadget_ep_queue()
587 u8 pending_empty = list_empty(&priv_ep->pending_req_list); in cdns3_wa2_gadget_ep_queue()
588 u8 descmiss_empty = list_empty(&priv_ep->wa2_descmiss_req_list); in cdns3_wa2_gadget_ep_queue()
596 !(priv_req->flags & REQUEST_INTERNAL)) { in cdns3_wa2_gadget_ep_queue()
598 &priv_req->request); in cdns3_wa2_gadget_ep_queue()
602 list_add_tail(&priv_req->request.list, in cdns3_wa2_gadget_ep_queue()
603 &priv_ep->pending_req_list); in cdns3_wa2_gadget_ep_queue()
605 priv_req->request.status); in cdns3_wa2_gadget_ep_queue()
624 if (priv_req->flags & REQUEST_INTERNAL) in cdns3_wa2_gadget_ep_queue()
625 list_add_tail(&priv_req->list, in cdns3_wa2_gadget_ep_queue()
626 &priv_ep->wa2_descmiss_req_list); in cdns3_wa2_gadget_ep_queue()
636 while (!list_empty(&priv_ep->wa2_descmiss_req_list)) { in cdns3_wa2_remove_old_request()
639 priv_req = cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list); in cdns3_wa2_remove_old_request()
640 chain = !!(priv_req->flags & REQUEST_INTERNAL_CH); in cdns3_wa2_remove_old_request()
644 kfree(priv_req->request.buf); in cdns3_wa2_remove_old_request()
645 list_del_init(&priv_req->list); in cdns3_wa2_remove_old_request()
646 cdns3_gadget_ep_free_request(&priv_ep->endpoint, in cdns3_wa2_remove_old_request()
647 &priv_req->request); in cdns3_wa2_remove_old_request()
648 --priv_ep->wa2_counter; in cdns3_wa2_remove_old_request()
656 * cdns3_wa2_descmissing_packet - handles descriptor missing event.
666 u8 pending_empty = list_empty(&priv_ep->pending_req_list); in cdns3_wa2_descmissing_packet()
674 if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_DET) { in cdns3_wa2_descmissing_packet()
675 priv_ep->flags &= ~EP_QUIRK_EXTRA_BUF_DET; in cdns3_wa2_descmissing_packet()
676 priv_ep->flags |= EP_QUIRK_EXTRA_BUF_EN; in cdns3_wa2_descmissing_packet()
681 if (priv_ep->wa2_counter >= CDNS3_WA2_NUM_BUFFERS) { in cdns3_wa2_descmissing_packet()
686 request = cdns3_gadget_ep_alloc_request(&priv_ep->endpoint, in cdns3_wa2_descmissing_packet()
692 priv_req->flags |= REQUEST_INTERNAL; in cdns3_wa2_descmissing_packet()
700 if (priv_ep->descmis_req) in cdns3_wa2_descmissing_packet()
701 priv_ep->descmis_req->flags |= REQUEST_INTERNAL_CH; in cdns3_wa2_descmissing_packet()
703 priv_req->request.buf = kzalloc(CDNS3_DESCMIS_BUF_SIZE, in cdns3_wa2_descmissing_packet()
705 priv_ep->wa2_counter++; in cdns3_wa2_descmissing_packet()
707 if (!priv_req->request.buf) { in cdns3_wa2_descmissing_packet()
708 cdns3_gadget_ep_free_request(&priv_ep->endpoint, request); in cdns3_wa2_descmissing_packet()
712 priv_req->request.length = CDNS3_DESCMIS_BUF_SIZE; in cdns3_wa2_descmissing_packet()
713 priv_ep->descmis_req = priv_req; in cdns3_wa2_descmissing_packet()
715 __cdns3_gadget_ep_queue(&priv_ep->endpoint, in cdns3_wa2_descmissing_packet()
716 &priv_ep->descmis_req->request, in cdns3_wa2_descmissing_packet()
722 dev_err(priv_ep->cdns3_dev->dev, in cdns3_wa2_descmissing_packet()
728 u16 tdl = EP_CMD_TDL_GET(readl(&priv_dev->regs->ep_cmd)); in cdns3_wa2_reset_tdl()
731 u16 reset_val = EP_CMD_TDL_MAX + 1 - tdl; in cdns3_wa2_reset_tdl()
734 &priv_dev->regs->ep_cmd); in cdns3_wa2_reset_tdl()
742 /* select EP0-out */ in cdns3_wa2_check_outq_status()
745 ep_sts_reg = readl(&priv_dev->regs->ep_sts); in cdns3_wa2_check_outq_status()
749 struct cdns3_endpoint *outq_ep = priv_dev->eps[outq_ep_num]; in cdns3_wa2_check_outq_status()
751 if ((outq_ep->flags & EP_ENABLED) && !(outq_ep->use_streams) && in cdns3_wa2_check_outq_status()
752 outq_ep->type != USB_ENDPOINT_XFER_ISOC && outq_ep_num) { in cdns3_wa2_check_outq_status()
753 u8 pending_empty = list_empty(&outq_ep->pending_req_list); in cdns3_wa2_check_outq_status()
755 if ((outq_ep->flags & EP_QUIRK_EXTRA_BUF_DET) || in cdns3_wa2_check_outq_status()
756 (outq_ep->flags & EP_QUIRK_EXTRA_BUF_EN) || in cdns3_wa2_check_outq_status()
762 cdns3_select_ep(priv_dev, outq_ep->num | in cdns3_wa2_check_outq_status()
763 outq_ep->dir); in cdns3_wa2_check_outq_status()
764 ep_sts_en_reg = readl(&priv_dev->regs->ep_sts_en); in cdns3_wa2_check_outq_status()
765 ep_cmd_reg = readl(&priv_dev->regs->ep_cmd); in cdns3_wa2_check_outq_status()
767 outq_ep->flags |= EP_TDLCHK_EN; in cdns3_wa2_check_outq_status()
768 cdns3_set_register_bit(&priv_dev->regs->ep_cfg, in cdns3_wa2_check_outq_status()
774 &priv_dev->regs->ep_sts_en); in cdns3_wa2_check_outq_status()
778 * Memory barrier - Reset tdl before ringing the in cdns3_wa2_check_outq_status()
788 * ring doorbell to generate DESCMIS irq in cdns3_wa2_check_outq_status()
791 &priv_dev->regs->ep_cmd); in cdns3_wa2_check_outq_status()
799 * cdns3_gadget_giveback - call struct usb_request's ->complete callback
805 * function will unmap @req and call its ->complete() callback to notify upper
812 struct cdns3_device *priv_dev = priv_ep->cdns3_dev; in cdns3_gadget_giveback()
813 struct usb_request *request = &priv_req->request; in cdns3_gadget_giveback()
815 list_del_init(&request->list); in cdns3_gadget_giveback()
817 if (request->status == -EINPROGRESS) in cdns3_gadget_giveback()
818 request->status = status; in cdns3_gadget_giveback()
820 usb_gadget_unmap_request_by_dev(priv_dev->sysdev, request, in cdns3_gadget_giveback()
821 priv_ep->dir); in cdns3_gadget_giveback()
823 if ((priv_req->flags & REQUEST_UNALIGNED) && in cdns3_gadget_giveback()
824 priv_ep->dir == USB_DIR_OUT && !request->status) in cdns3_gadget_giveback()
825 memcpy(request->buf, priv_req->aligned_buf->buf, in cdns3_gadget_giveback()
826 request->length); in cdns3_gadget_giveback()
828 priv_req->flags &= ~(REQUEST_PENDING | REQUEST_UNALIGNED); in cdns3_gadget_giveback()
830 priv_req->finished_trb = 0; in cdns3_gadget_giveback()
833 if (priv_dev->dev_ver < DEV_VER_V2) { in cdns3_gadget_giveback()
840 if (request->complete) { in cdns3_gadget_giveback()
841 spin_unlock(&priv_dev->lock); in cdns3_gadget_giveback()
842 usb_gadget_giveback_request(&priv_ep->endpoint, in cdns3_gadget_giveback()
844 spin_lock(&priv_dev->lock); in cdns3_gadget_giveback()
847 if (request->buf == priv_dev->zlp_buf) in cdns3_gadget_giveback()
848 cdns3_gadget_ep_free_request(&priv_ep->endpoint, request); in cdns3_gadget_giveback()
854 if (priv_ep->wa1_set) { in cdns3_wa1_restore_cycle_bit()
857 priv_ep->wa1_set = 0; in cdns3_wa1_restore_cycle_bit()
858 priv_ep->wa1_trb_index = 0xFFFF; in cdns3_wa1_restore_cycle_bit()
859 if (priv_ep->wa1_cycle_bit) { in cdns3_wa1_restore_cycle_bit()
860 priv_ep->wa1_trb->control = in cdns3_wa1_restore_cycle_bit()
861 priv_ep->wa1_trb->control | cpu_to_le32(0x1); in cdns3_wa1_restore_cycle_bit()
863 priv_ep->wa1_trb->control = in cdns3_wa1_restore_cycle_bit()
864 priv_ep->wa1_trb->control & cpu_to_le32(~0x1); in cdns3_wa1_restore_cycle_bit()
876 spin_lock_irqsave(&priv_dev->lock, flags); in cdns3_free_aligned_request_buf()
878 list_for_each_entry_safe(buf, tmp, &priv_dev->aligned_buf_list, list) { in cdns3_free_aligned_request_buf()
879 if (!buf->in_use) { in cdns3_free_aligned_request_buf()
880 list_del(&buf->list); in cdns3_free_aligned_request_buf()
883 * Re-enable interrupts to free DMA capable memory. in cdns3_free_aligned_request_buf()
887 spin_unlock_irqrestore(&priv_dev->lock, flags); in cdns3_free_aligned_request_buf()
888 dma_free_coherent(priv_dev->sysdev, buf->size, in cdns3_free_aligned_request_buf()
889 buf->buf, buf->dma); in cdns3_free_aligned_request_buf()
891 spin_lock_irqsave(&priv_dev->lock, flags); in cdns3_free_aligned_request_buf()
895 spin_unlock_irqrestore(&priv_dev->lock, flags); in cdns3_free_aligned_request_buf()
900 struct cdns3_endpoint *priv_ep = priv_req->priv_ep; in cdns3_prepare_aligned_request_buf()
901 struct cdns3_device *priv_dev = priv_ep->cdns3_dev; in cdns3_prepare_aligned_request_buf()
905 if (!((uintptr_t)priv_req->request.buf & 0x7)) in cdns3_prepare_aligned_request_buf()
908 buf = priv_req->aligned_buf; in cdns3_prepare_aligned_request_buf()
910 if (!buf || priv_req->request.length > buf->size) { in cdns3_prepare_aligned_request_buf()
913 return -ENOMEM; in cdns3_prepare_aligned_request_buf()
915 buf->size = priv_req->request.length; in cdns3_prepare_aligned_request_buf()
917 buf->buf = dma_alloc_coherent(priv_dev->sysdev, in cdns3_prepare_aligned_request_buf()
918 buf->size, in cdns3_prepare_aligned_request_buf()
919 &buf->dma, in cdns3_prepare_aligned_request_buf()
921 if (!buf->buf) { in cdns3_prepare_aligned_request_buf()
923 return -ENOMEM; in cdns3_prepare_aligned_request_buf()
926 if (priv_req->aligned_buf) { in cdns3_prepare_aligned_request_buf()
928 priv_req->aligned_buf->in_use = 0; in cdns3_prepare_aligned_request_buf()
930 &priv_dev->aligned_buf_wq); in cdns3_prepare_aligned_request_buf()
933 buf->in_use = 1; in cdns3_prepare_aligned_request_buf()
934 priv_req->aligned_buf = buf; in cdns3_prepare_aligned_request_buf()
936 list_add_tail(&buf->list, in cdns3_prepare_aligned_request_buf()
937 &priv_dev->aligned_buf_list); in cdns3_prepare_aligned_request_buf()
940 if (priv_ep->dir == USB_DIR_IN) { in cdns3_prepare_aligned_request_buf()
941 memcpy(buf->buf, priv_req->request.buf, in cdns3_prepare_aligned_request_buf()
942 priv_req->request.length); in cdns3_prepare_aligned_request_buf()
945 priv_req->flags |= REQUEST_UNALIGNED; in cdns3_prepare_aligned_request_buf()
954 struct cdns3_device *priv_dev = priv_ep->cdns3_dev; in cdns3_wa1_update_guard()
956 if (!priv_ep->wa1_set) { in cdns3_wa1_update_guard()
959 doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY); in cdns3_wa1_update_guard()
962 priv_ep->wa1_cycle_bit = priv_ep->pcs ? TRB_CYCLE : 0; in cdns3_wa1_update_guard()
963 priv_ep->wa1_set = 1; in cdns3_wa1_update_guard()
964 priv_ep->wa1_trb = trb; in cdns3_wa1_update_guard()
965 priv_ep->wa1_trb_index = priv_ep->enqueue; in cdns3_wa1_update_guard()
979 doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY); in cdns3_wa1_tray_restore_cycle_bit()
982 if (!doorbell || dma_index != priv_ep->wa1_trb_index) in cdns3_wa1_tray_restore_cycle_bit()
989 struct cdns3_device *priv_dev = priv_ep->cdns3_dev; in cdns3_ep_run_stream_transfer()
997 unsigned int sg_idx = priv_ep->stream_sg_idx; in cdns3_ep_run_stream_transfer()
1000 address = priv_ep->endpoint.desc->bEndpointAddress; in cdns3_ep_run_stream_transfer()
1002 priv_ep->flags |= EP_PENDING_REQUEST; in cdns3_ep_run_stream_transfer()
1005 if (priv_req->flags & REQUEST_UNALIGNED) in cdns3_ep_run_stream_transfer()
1006 trb_dma = priv_req->aligned_buf->dma; in cdns3_ep_run_stream_transfer()
1008 trb_dma = request->dma; in cdns3_ep_run_stream_transfer()
1011 trb = priv_ep->trb_pool + priv_ep->enqueue; in cdns3_ep_run_stream_transfer()
1012 priv_req->start_trb = priv_ep->enqueue; in cdns3_ep_run_stream_transfer()
1013 priv_req->end_trb = priv_req->start_trb; in cdns3_ep_run_stream_transfer()
1014 priv_req->trb = trb; in cdns3_ep_run_stream_transfer()
1016 cdns3_select_ep(priv_ep->cdns3_dev, address); in cdns3_ep_run_stream_transfer()
1019 TRB_STREAM_ID(priv_req->request.stream_id) | TRB_ISP; in cdns3_ep_run_stream_transfer()
1021 if (!request->num_sgs) { in cdns3_ep_run_stream_transfer()
1022 trb->buffer = cpu_to_le32(TRB_BUFFER(trb_dma)); in cdns3_ep_run_stream_transfer()
1023 length = request->length; in cdns3_ep_run_stream_transfer()
1025 trb->buffer = cpu_to_le32(TRB_BUFFER(request->sg[sg_idx].dma_address)); in cdns3_ep_run_stream_transfer()
1026 length = request->sg[sg_idx].length; in cdns3_ep_run_stream_transfer()
1029 tdl = DIV_ROUND_UP(length, priv_ep->endpoint.maxpacket); in cdns3_ep_run_stream_transfer()
1031 trb->length = cpu_to_le32(TRB_BURST_LEN(16) | TRB_LEN(length)); in cdns3_ep_run_stream_transfer()
1038 if (priv_dev->dev_ver >= DEV_VER_V2) { in cdns3_ep_run_stream_transfer()
1039 if (priv_dev->gadget.speed == USB_SPEED_SUPER) in cdns3_ep_run_stream_transfer()
1040 trb->length |= cpu_to_le32(TRB_TDL_SS_SIZE(tdl)); in cdns3_ep_run_stream_transfer()
1042 priv_req->flags |= REQUEST_PENDING; in cdns3_ep_run_stream_transfer()
1044 trb->control = cpu_to_le32(control); in cdns3_ep_run_stream_transfer()
1046 trace_cdns3_prepare_trb(priv_ep, priv_req->trb); in cdns3_ep_run_stream_transfer()
1049 * Memory barrier - Cycle Bit must be set before trb->length and in cdns3_ep_run_stream_transfer()
1050 * trb->buffer fields. in cdns3_ep_run_stream_transfer()
1055 writel(EP_TRADDR_TRADDR(priv_ep->trb_pool_dma), in cdns3_ep_run_stream_transfer()
1056 &priv_dev->regs->ep_traddr); in cdns3_ep_run_stream_transfer()
1058 if (!(priv_ep->flags & EP_STALLED)) { in cdns3_ep_run_stream_transfer()
1061 writel(EP_STS_TRBERR | EP_STS_DESCMIS, &priv_dev->regs->ep_sts); in cdns3_ep_run_stream_transfer()
1063 priv_ep->prime_flag = false; in cdns3_ep_run_stream_transfer()
1070 if (priv_dev->dev_ver < DEV_VER_V2) in cdns3_ep_run_stream_transfer()
1072 &priv_dev->regs->ep_cmd); in cdns3_ep_run_stream_transfer()
1073 else if (priv_dev->dev_ver > DEV_VER_V2) in cdns3_ep_run_stream_transfer()
1074 writel(tdl, &priv_dev->regs->ep_tdl); in cdns3_ep_run_stream_transfer()
1076 priv_ep->last_stream_id = priv_req->request.stream_id; in cdns3_ep_run_stream_transfer()
1077 writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd); in cdns3_ep_run_stream_transfer()
1078 writel(EP_CMD_ERDY_SID(priv_req->request.stream_id) | in cdns3_ep_run_stream_transfer()
1079 EP_CMD_ERDY, &priv_dev->regs->ep_cmd); in cdns3_ep_run_stream_transfer()
1081 trace_cdns3_doorbell_epx(priv_ep->name, in cdns3_ep_run_stream_transfer()
1082 readl(&priv_dev->regs->ep_traddr)); in cdns3_ep_run_stream_transfer()
1093 struct cdns3_device *priv_dev = priv_ep->cdns3_dev; in cdns3_rearm_drdy_if_needed()
1095 if (priv_dev->dev_ver < DEV_VER_V3) in cdns3_rearm_drdy_if_needed()
1098 if (readl(&priv_dev->regs->ep_sts) & EP_STS_TRBERR) { in cdns3_rearm_drdy_if_needed()
1099 writel(EP_STS_TRBERR, &priv_dev->regs->ep_sts); in cdns3_rearm_drdy_if_needed()
1100 writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd); in cdns3_rearm_drdy_if_needed()
1105 * cdns3_ep_run_transfer - start transfer on no-default endpoint hardware
1114 struct cdns3_device *priv_dev = priv_ep->cdns3_dev; in cdns3_ep_run_transfer()
1127 bool sg_supported = !!(request->num_mapped_sgs); in cdns3_ep_run_transfer()
1129 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC) in cdns3_ep_run_transfer()
1130 num_trb = priv_ep->interval; in cdns3_ep_run_transfer()
1132 num_trb = sg_supported ? request->num_mapped_sgs : 1; in cdns3_ep_run_transfer()
1134 if (num_trb > priv_ep->free_trbs) { in cdns3_ep_run_transfer()
1135 priv_ep->flags |= EP_RING_FULL; in cdns3_ep_run_transfer()
1136 return -ENOBUFS; in cdns3_ep_run_transfer()
1140 address = priv_ep->endpoint.desc->bEndpointAddress; in cdns3_ep_run_transfer()
1142 priv_ep->flags |= EP_PENDING_REQUEST; in cdns3_ep_run_transfer()
1145 if (priv_req->flags & REQUEST_UNALIGNED) in cdns3_ep_run_transfer()
1146 trb_dma = priv_req->aligned_buf->dma; in cdns3_ep_run_transfer()
1148 trb_dma = request->dma; in cdns3_ep_run_transfer()
1150 trb = priv_ep->trb_pool + priv_ep->enqueue; in cdns3_ep_run_transfer()
1151 priv_req->start_trb = priv_ep->enqueue; in cdns3_ep_run_transfer()
1152 priv_req->trb = trb; in cdns3_ep_run_transfer()
1154 cdns3_select_ep(priv_ep->cdns3_dev, address); in cdns3_ep_run_transfer()
1156 /* prepare ring */ in cdns3_ep_run_transfer()
1157 if ((priv_ep->enqueue + num_trb) >= (priv_ep->num_trbs - 1)) { in cdns3_ep_run_transfer()
1161 doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY); in cdns3_ep_run_transfer()
1165 if (doorbell && dma_index == priv_ep->num_trbs - 1) { in cdns3_ep_run_transfer()
1166 priv_ep->flags |= EP_DEFERRED_DRDY; in cdns3_ep_run_transfer()
1167 return -ENOBUFS; in cdns3_ep_run_transfer()
1171 link_trb = priv_ep->trb_pool + (priv_ep->num_trbs - 1); in cdns3_ep_run_transfer()
1180 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC || in cdns3_ep_run_transfer()
1184 link_trb->control = cpu_to_le32(((priv_ep->pcs) ? TRB_CYCLE : 0) | in cdns3_ep_run_transfer()
1188 if (priv_dev->dev_ver <= DEV_VER_V2) in cdns3_ep_run_transfer()
1192 s = request->sg; in cdns3_ep_run_transfer()
1195 control = priv_ep->pcs ? 0 : TRB_CYCLE; in cdns3_ep_run_transfer()
1196 trb->length = 0; in cdns3_ep_run_transfer()
1197 if (priv_dev->dev_ver >= DEV_VER_V2) { in cdns3_ep_run_transfer()
1200 td_size = DIV_ROUND_UP(request->length, in cdns3_ep_run_transfer()
1201 priv_ep->endpoint.maxpacket); in cdns3_ep_run_transfer()
1202 if (priv_dev->gadget.speed == USB_SPEED_SUPER) in cdns3_ep_run_transfer()
1203 trb->length = TRB_TDL_SS_SIZE(td_size); in cdns3_ep_run_transfer()
1214 trb->buffer = cpu_to_le32(TRB_BUFFER(sg_dma_address(s))); in cdns3_ep_run_transfer()
1217 trb->buffer = cpu_to_le32(TRB_BUFFER(trb_dma)); in cdns3_ep_run_transfer()
1218 length = request->length; in cdns3_ep_run_transfer()
1221 if (priv_ep->flags & EP_TDLCHK_EN) in cdns3_ep_run_transfer()
1223 priv_ep->endpoint.maxpacket); in cdns3_ep_run_transfer()
1225 trb->length |= cpu_to_le32(TRB_BURST_LEN(priv_ep->trb_burst_size) | in cdns3_ep_run_transfer()
1227 pcs = priv_ep->pcs ? TRB_CYCLE : 0; in cdns3_ep_run_transfer()
1236 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && !priv_ep->dir) { in cdns3_ep_run_transfer()
1240 if (sg_iter == (num_trb - 1) && sg_iter != 0) in cdns3_ep_run_transfer()
1245 trb->control = cpu_to_le32(control); in cdns3_ep_run_transfer()
1247 priv_req->trb->control = cpu_to_le32(control); in cdns3_ep_run_transfer()
1250 trb->control |= TRB_ISP; in cdns3_ep_run_transfer()
1252 if (sg_iter < num_trb - 1) in cdns3_ep_run_transfer()
1253 trb->control |= TRB_CHAIN; in cdns3_ep_run_transfer()
1260 priv_req->end_trb = priv_ep->enqueue; in cdns3_ep_run_transfer()
1262 trb = priv_ep->trb_pool + priv_ep->enqueue; in cdns3_ep_run_transfer()
1263 trb->length = 0; in cdns3_ep_run_transfer()
1266 trb = priv_req->trb; in cdns3_ep_run_transfer()
1268 priv_req->flags |= REQUEST_PENDING; in cdns3_ep_run_transfer()
1269 priv_req->num_of_trb = num_trb; in cdns3_ep_run_transfer()
1272 trb->control |= cpu_to_le32(TRB_IOC | TRB_ISP); in cdns3_ep_run_transfer()
1274 if (priv_dev->dev_ver < DEV_VER_V2 && in cdns3_ep_run_transfer()
1275 (priv_ep->flags & EP_TDLCHK_EN)) { in cdns3_ep_run_transfer()
1277 u16 old_tdl = EP_CMD_TDL_GET(readl(&priv_dev->regs->ep_cmd)); in cdns3_ep_run_transfer()
1281 priv_ep->pending_tdl = total_tdl - EP_CMD_TDL_MAX; in cdns3_ep_run_transfer()
1285 tdl -= old_tdl; in cdns3_ep_run_transfer()
1287 &priv_dev->regs->ep_cmd); in cdns3_ep_run_transfer()
1292 * Memory barrier - cycle bit must be set before other filds in trb. in cdns3_ep_run_transfer()
1298 trb->control = trb->control ^ cpu_to_le32(1); in cdns3_ep_run_transfer()
1300 if (priv_dev->dev_ver <= DEV_VER_V2) in cdns3_ep_run_transfer()
1309 trb = priv_ep->trb_pool; in cdns3_ep_run_transfer()
1310 num_trb = num_trb - i; in cdns3_ep_run_transfer()
1317 trace_cdns3_prepare_trb(priv_ep, priv_req->trb); in cdns3_ep_run_transfer()
1321 * Memory barrier - Cycle Bit must be set before trb->length and in cdns3_ep_run_transfer()
1322 * trb->buffer fields. in cdns3_ep_run_transfer()
1327 * For DMULT mode we can set address to transfer ring only once after in cdns3_ep_run_transfer()
1330 if (priv_ep->flags & EP_UPDATE_EP_TRBADDR) { in cdns3_ep_run_transfer()
1336 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && !priv_ep->dir && in cdns3_ep_run_transfer()
1337 !(priv_ep->flags & EP_QUIRK_ISO_OUT_EN)) { in cdns3_ep_run_transfer()
1338 priv_ep->flags |= EP_QUIRK_ISO_OUT_EN; in cdns3_ep_run_transfer()
1339 cdns3_set_register_bit(&priv_dev->regs->ep_cfg, in cdns3_ep_run_transfer()
1343 writel(EP_TRADDR_TRADDR(priv_ep->trb_pool_dma + in cdns3_ep_run_transfer()
1344 priv_req->start_trb * TRB_SIZE), in cdns3_ep_run_transfer()
1345 &priv_dev->regs->ep_traddr); in cdns3_ep_run_transfer()
1347 priv_ep->flags &= ~EP_UPDATE_EP_TRBADDR; in cdns3_ep_run_transfer()
1350 if (!priv_ep->wa1_set && !(priv_ep->flags & EP_STALLED)) { in cdns3_ep_run_transfer()
1353 writel(EP_STS_TRBERR | EP_STS_DESCMIS, &priv_dev->regs->ep_sts); in cdns3_ep_run_transfer()
1354 writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd); in cdns3_ep_run_transfer()
1356 trace_cdns3_doorbell_epx(priv_ep->name, in cdns3_ep_run_transfer()
1357 readl(&priv_dev->regs->ep_traddr)); in cdns3_ep_run_transfer()
1371 if (priv_dev->hw_configured_flag) in cdns3_set_hw_configuration()
1374 writel(USB_CONF_CFGSET, &priv_dev->regs->usb_conf); in cdns3_set_hw_configuration()
1376 cdns3_set_register_bit(&priv_dev->regs->usb_conf, in cdns3_set_hw_configuration()
1379 priv_dev->hw_configured_flag = 1; in cdns3_set_hw_configuration()
1381 list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) { in cdns3_set_hw_configuration()
1382 if (ep->enabled) { in cdns3_set_hw_configuration()
1392 * cdns3_trb_handled - check whether trb has been handled by DMA
1401 * SR - start ring
1402 * ER - end ring
1403 * DQ = priv_ep->dequeue - dequeue position
1404 * EQ = priv_ep->enqueue - enqueue position
1405 * ST = priv_req->start_trb - index of first TRB in transfer ring
1406 * ET = priv_req->end_trb - index of last TRB in transfer ring
1407 * CI = current_index - index of processed TRB by DMA.
1410 * Then, we check if cycle bit for index priv_ep->dequeue
1414 * 1. priv_ep->dequeue never equals to current_index.
1415 * 2 priv_ep->enqueue never exceed priv_ep->dequeue
1416 * 3. exception: priv_ep->enqueue == priv_ep->dequeue
1417 * and priv_ep->free_trbs is zero.
1421 * Case 1 - priv_ep->dequeue < current_index
1425 * Case 2 - priv_ep->dequeue > current_index
1427 * transfer ring.
1433 struct cdns3_device *priv_dev = priv_ep->cdns3_dev; in cdns3_trb_handled()
1440 doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY); in cdns3_trb_handled()
1443 if (priv_req->start_trb < priv_req->end_trb) { in cdns3_trb_handled()
1444 if (priv_ep->dequeue > priv_req->end_trb) in cdns3_trb_handled()
1447 if (priv_ep->dequeue < priv_req->start_trb) in cdns3_trb_handled()
1451 if ((priv_req->start_trb > priv_req->end_trb) && in cdns3_trb_handled()
1452 (priv_ep->dequeue > priv_req->end_trb) && in cdns3_trb_handled()
1453 (priv_ep->dequeue < priv_req->start_trb)) in cdns3_trb_handled()
1456 if ((priv_req->start_trb == priv_req->end_trb) && in cdns3_trb_handled()
1457 (priv_ep->dequeue != priv_req->end_trb)) in cdns3_trb_handled()
1460 trb = &priv_ep->trb_pool[priv_ep->dequeue]; in cdns3_trb_handled()
1462 if ((le32_to_cpu(trb->control) & TRB_CYCLE) != priv_ep->ccs) in cdns3_trb_handled()
1465 if (doorbell == 1 && current_index == priv_ep->dequeue) in cdns3_trb_handled()
1469 if (TRBS_PER_SEGMENT == 2 && priv_ep->type != USB_ENDPOINT_XFER_ISOC) { in cdns3_trb_handled()
1474 if (priv_ep->enqueue == priv_ep->dequeue && in cdns3_trb_handled()
1475 priv_ep->free_trbs == 0) { in cdns3_trb_handled()
1477 } else if (priv_ep->dequeue < current_index) { in cdns3_trb_handled()
1478 if ((current_index == (priv_ep->num_trbs - 1)) && in cdns3_trb_handled()
1479 !priv_ep->dequeue) in cdns3_trb_handled()
1483 } else if (priv_ep->dequeue > current_index) { in cdns3_trb_handled()
1502 while (!list_empty(&priv_ep->pending_req_list)) { in cdns3_transfer_completed()
1503 request = cdns3_next_request(&priv_ep->pending_req_list); in cdns3_transfer_completed()
1506 trb = priv_ep->trb_pool + priv_ep->dequeue; in cdns3_transfer_completed()
1509 while (TRB_FIELD_TO_TYPE(le32_to_cpu(trb->control)) == TRB_LINK) { in cdns3_transfer_completed()
1512 trb = priv_ep->trb_pool + priv_ep->dequeue; in cdns3_transfer_completed()
1515 if (!request->stream_id) { in cdns3_transfer_completed()
1516 /* Re-select endpoint. It could be changed by other CPU in cdns3_transfer_completed()
1519 cdns3_select_ep(priv_dev, priv_ep->endpoint.address); in cdns3_transfer_completed()
1522 priv_req->finished_trb++; in cdns3_transfer_completed()
1523 if (priv_req->finished_trb >= priv_req->num_of_trb) in cdns3_transfer_completed()
1526 trb = priv_ep->trb_pool + priv_ep->dequeue; in cdns3_transfer_completed()
1530 request->actual += in cdns3_transfer_completed()
1531 TRB_LEN(le32_to_cpu(trb->length)); in cdns3_transfer_completed()
1533 if (priv_req->num_of_trb > 1 && in cdns3_transfer_completed()
1534 le32_to_cpu(trb->control) & TRB_SMM && in cdns3_transfer_completed()
1535 le32_to_cpu(trb->control) & TRB_CHAIN) in cdns3_transfer_completed()
1549 if (priv_ep->type != USB_ENDPOINT_XFER_ISOC && in cdns3_transfer_completed()
1553 /* Re-select endpoint. It could be changed by other CPU in cdns3_transfer_completed()
1556 cdns3_select_ep(priv_dev, priv_ep->endpoint.address); in cdns3_transfer_completed()
1558 trb = priv_ep->trb_pool; in cdns3_transfer_completed()
1561 if (trb != priv_req->trb) in cdns3_transfer_completed()
1562 dev_warn(priv_dev->dev, in cdns3_transfer_completed()
1564 priv_req->trb, trb); in cdns3_transfer_completed()
1566 request->actual += TRB_LEN(le32_to_cpu(trb->length)); in cdns3_transfer_completed()
1568 if (!request->num_sgs || in cdns3_transfer_completed()
1569 (request->num_sgs == (priv_ep->stream_sg_idx + 1))) { in cdns3_transfer_completed()
1570 priv_ep->stream_sg_idx = 0; in cdns3_transfer_completed()
1573 priv_ep->stream_sg_idx++; in cdns3_transfer_completed()
1579 priv_ep->flags &= ~EP_PENDING_REQUEST; in cdns3_transfer_completed()
1582 if (!(priv_ep->flags & EP_STALLED) && in cdns3_transfer_completed()
1583 !(priv_ep->flags & EP_STALL_PENDING)) in cdns3_transfer_completed()
1589 struct cdns3_device *priv_dev = priv_ep->cdns3_dev; in cdns3_rearm_transfer()
1598 writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd); in cdns3_rearm_transfer()
1602 trace_cdns3_doorbell_epx(priv_ep->name, in cdns3_rearm_transfer()
1603 readl(&priv_dev->regs->ep_traddr)); in cdns3_rearm_transfer()
1609 u16 tdl = priv_ep->pending_tdl; in cdns3_reprogram_tdl()
1610 struct cdns3_device *priv_dev = priv_ep->cdns3_dev; in cdns3_reprogram_tdl()
1614 priv_ep->pending_tdl -= EP_CMD_TDL_MAX; in cdns3_reprogram_tdl()
1616 priv_ep->pending_tdl = 0; in cdns3_reprogram_tdl()
1619 writel(EP_CMD_TDL_SET(tdl) | EP_CMD_STDL, &priv_dev->regs->ep_cmd); in cdns3_reprogram_tdl()
1623 * cdns3_check_ep_interrupt_proceed - Processes interrupt related to endpoint
1630 struct cdns3_device *priv_dev = priv_ep->cdns3_dev; in cdns3_check_ep_interrupt_proceed()
1636 cdns3_select_ep(priv_dev, priv_ep->endpoint.address); in cdns3_check_ep_interrupt_proceed()
1640 ep_sts_reg = readl(&priv_dev->regs->ep_sts); in cdns3_check_ep_interrupt_proceed()
1641 writel(ep_sts_reg, &priv_dev->regs->ep_sts); in cdns3_check_ep_interrupt_proceed()
1643 if ((ep_sts_reg & EP_STS_PRIME) && priv_ep->use_streams) { in cdns3_check_ep_interrupt_proceed()
1658 EP_CMD_ERDY_SID(priv_ep->last_stream_id), in cdns3_check_ep_interrupt_proceed()
1659 &priv_dev->regs->ep_cmd); in cdns3_check_ep_interrupt_proceed()
1662 priv_ep->prime_flag = true; in cdns3_check_ep_interrupt_proceed()
1664 pending_request = cdns3_next_request(&priv_ep->pending_req_list); in cdns3_check_ep_interrupt_proceed()
1665 deferred_request = cdns3_next_request(&priv_ep->deferred_req_list); in cdns3_check_ep_interrupt_proceed()
1674 if (priv_ep->flags & EP_STALL_PENDING && in cdns3_check_ep_interrupt_proceed()
1676 priv_dev->dev_ver < DEV_VER_V2)) { in cdns3_check_ep_interrupt_proceed()
1683 * OUT data packet. If host disable stream or lost some packet in cdns3_check_ep_interrupt_proceed()
1687 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && in cdns3_check_ep_interrupt_proceed()
1688 !priv_ep->wa1_set) { in cdns3_check_ep_interrupt_proceed()
1689 if (!priv_ep->dir) { in cdns3_check_ep_interrupt_proceed()
1690 u32 ep_cfg = readl(&priv_dev->regs->ep_cfg); in cdns3_check_ep_interrupt_proceed()
1693 writel(ep_cfg, &priv_dev->regs->ep_cfg); in cdns3_check_ep_interrupt_proceed()
1694 priv_ep->flags &= ~EP_QUIRK_ISO_OUT_EN; in cdns3_check_ep_interrupt_proceed()
1695 priv_ep->flags |= EP_UPDATE_EP_TRBADDR; in cdns3_check_ep_interrupt_proceed()
1698 } else if (!(priv_ep->flags & EP_STALLED) && in cdns3_check_ep_interrupt_proceed()
1699 !(priv_ep->flags & EP_STALL_PENDING)) { in cdns3_check_ep_interrupt_proceed()
1700 if (priv_ep->flags & EP_DEFERRED_DRDY) { in cdns3_check_ep_interrupt_proceed()
1701 priv_ep->flags &= ~EP_DEFERRED_DRDY; in cdns3_check_ep_interrupt_proceed()
1705 priv_ep->wa1_set); in cdns3_check_ep_interrupt_proceed()
1712 if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN) { in cdns3_check_ep_interrupt_proceed()
1714 priv_ep->flags |= EP_QUIRK_END_TRANSFER; in cdns3_check_ep_interrupt_proceed()
1716 priv_ep->flags &= ~EP_QUIRK_END_TRANSFER; in cdns3_check_ep_interrupt_proceed()
1719 if (!priv_ep->use_streams) { in cdns3_check_ep_interrupt_proceed()
1723 } else if ((priv_ep->flags & EP_TDLCHK_EN) & in cdns3_check_ep_interrupt_proceed()
1724 priv_ep->pending_tdl) { in cdns3_check_ep_interrupt_proceed()
1728 } else if (priv_ep->dir == USB_DIR_OUT) { in cdns3_check_ep_interrupt_proceed()
1729 priv_ep->ep_sts_pending |= ep_sts_reg; in cdns3_check_ep_interrupt_proceed()
1739 if (priv_ep->dir == USB_DIR_OUT && (ep_sts_reg & EP_STS_MD_EXIT) && in cdns3_check_ep_interrupt_proceed()
1740 (priv_ep->ep_sts_pending & EP_STS_IOT) && priv_ep->use_streams) { in cdns3_check_ep_interrupt_proceed()
1741 priv_ep->ep_sts_pending = 0; in cdns3_check_ep_interrupt_proceed()
1747 * priv_ep->flags & EP_QUIRK_EXTRA_BUF_DET or in cdns3_check_ep_interrupt_proceed()
1748 * priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN. in cdns3_check_ep_interrupt_proceed()
1751 if (ep_sts_reg & EP_STS_DESCMIS && priv_dev->dev_ver < DEV_VER_V2 && in cdns3_check_ep_interrupt_proceed()
1752 !(priv_ep->flags & EP_STALLED)) in cdns3_check_ep_interrupt_proceed()
1760 if (priv_dev->gadget_driver && priv_dev->gadget_driver->disconnect) in cdns3_disconnect_gadget()
1761 priv_dev->gadget_driver->disconnect(&priv_dev->gadget); in cdns3_disconnect_gadget()
1765 * cdns3_check_usb_interrupt_proceed - Processes interrupt related to device
1772 __must_hold(&priv_dev->lock) in cdns3_check_usb_interrupt_proceed()
1783 if (readl(&priv_dev->regs->drbl)) in cdns3_check_usb_interrupt_proceed()
1790 priv_dev->gadget.speed = speed; in cdns3_check_usb_interrupt_proceed()
1791 usb_gadget_set_state(&priv_dev->gadget, USB_STATE_POWERED); in cdns3_check_usb_interrupt_proceed()
1797 spin_unlock(&priv_dev->lock); in cdns3_check_usb_interrupt_proceed()
1799 spin_lock(&priv_dev->lock); in cdns3_check_usb_interrupt_proceed()
1800 priv_dev->gadget.speed = USB_SPEED_UNKNOWN; in cdns3_check_usb_interrupt_proceed()
1801 usb_gadget_set_state(&priv_dev->gadget, USB_STATE_NOTATTACHED); in cdns3_check_usb_interrupt_proceed()
1806 if (priv_dev->gadget_driver && in cdns3_check_usb_interrupt_proceed()
1807 priv_dev->gadget_driver->suspend) { in cdns3_check_usb_interrupt_proceed()
1808 spin_unlock(&priv_dev->lock); in cdns3_check_usb_interrupt_proceed()
1809 priv_dev->gadget_driver->suspend(&priv_dev->gadget); in cdns3_check_usb_interrupt_proceed()
1810 spin_lock(&priv_dev->lock); in cdns3_check_usb_interrupt_proceed()
1815 if (priv_dev->gadget_driver && in cdns3_check_usb_interrupt_proceed()
1816 priv_dev->gadget_driver->resume) { in cdns3_check_usb_interrupt_proceed()
1817 spin_unlock(&priv_dev->lock); in cdns3_check_usb_interrupt_proceed()
1818 priv_dev->gadget_driver->resume(&priv_dev->gadget); in cdns3_check_usb_interrupt_proceed()
1819 spin_lock(&priv_dev->lock); in cdns3_check_usb_interrupt_proceed()
1825 if (priv_dev->gadget_driver) { in cdns3_check_usb_interrupt_proceed()
1826 spin_unlock(&priv_dev->lock); in cdns3_check_usb_interrupt_proceed()
1827 usb_gadget_udc_reset(&priv_dev->gadget, in cdns3_check_usb_interrupt_proceed()
1828 priv_dev->gadget_driver); in cdns3_check_usb_interrupt_proceed()
1829 spin_lock(&priv_dev->lock); in cdns3_check_usb_interrupt_proceed()
1833 priv_dev->gadget.speed = speed; in cdns3_check_usb_interrupt_proceed()
1841 * cdns3_device_irq_handler- interrupt handler for device part of controller
1851 struct cdns3 *cdns = dev_get_drvdata(priv_dev->dev); in cdns3_device_irq_handler()
1855 if (cdns->in_lpm) in cdns3_device_irq_handler()
1859 reg = readl(&priv_dev->regs->usb_ists); in cdns3_device_irq_handler()
1868 reg = ~reg & readl(&priv_dev->regs->usb_ien); in cdns3_device_irq_handler()
1870 writel(reg, &priv_dev->regs->usb_ien); in cdns3_device_irq_handler()
1875 reg = readl(&priv_dev->regs->ep_ists); in cdns3_device_irq_handler()
1877 writel(0, &priv_dev->regs->ep_ien); in cdns3_device_irq_handler()
1885 * cdns3_device_thread_irq_handler- interrupt handler for device part
1901 spin_lock_irqsave(&priv_dev->lock, flags); in cdns3_device_thread_irq_handler()
1903 reg = readl(&priv_dev->regs->usb_ists); in cdns3_device_thread_irq_handler()
1905 writel(reg, &priv_dev->regs->usb_ists); in cdns3_device_thread_irq_handler()
1906 writel(USB_IEN_INIT, &priv_dev->regs->usb_ien); in cdns3_device_thread_irq_handler()
1911 reg = readl(&priv_dev->regs->ep_ists); in cdns3_device_thread_irq_handler()
1932 cdns3_check_ep_interrupt_proceed(priv_dev->eps[bit]); in cdns3_device_thread_irq_handler()
1936 if (priv_dev->dev_ver < DEV_VER_V2 && priv_dev->using_streams) in cdns3_device_thread_irq_handler()
1940 writel(~0, &priv_dev->regs->ep_ien); in cdns3_device_thread_irq_handler()
1941 spin_unlock_irqrestore(&priv_dev->lock, flags); in cdns3_device_thread_irq_handler()
1947 * cdns3_ep_onchip_buffer_reserve - Try to reserve onchip buf for EP
1964 remained = priv_dev->onchip_buffers - priv_dev->onchip_used_size - 2; in cdns3_ep_onchip_buffer_reserve()
1968 return -EPERM; in cdns3_ep_onchip_buffer_reserve()
1970 priv_dev->onchip_used_size += size; in cdns3_ep_onchip_buffer_reserve()
1978 if (priv_dev->out_mem_is_allocated >= size) in cdns3_ep_onchip_buffer_reserve()
1981 required = size - priv_dev->out_mem_is_allocated; in cdns3_ep_onchip_buffer_reserve()
1984 return -EPERM; in cdns3_ep_onchip_buffer_reserve()
1986 priv_dev->out_mem_is_allocated += required; in cdns3_ep_onchip_buffer_reserve()
1987 priv_dev->onchip_used_size += required; in cdns3_ep_onchip_buffer_reserve()
1996 struct cdns3_usb_regs __iomem *regs = priv_dev->regs; in cdns3_configure_dmult()
1999 if (priv_dev->dev_ver <= DEV_VER_V2) in cdns3_configure_dmult()
2000 writel(USB_CONF_DMULT, ®s->usb_conf); in cdns3_configure_dmult()
2002 if (priv_dev->dev_ver == DEV_VER_V2) in cdns3_configure_dmult()
2003 writel(USB_CONF2_EN_TDL_TRB, ®s->usb_conf2); in cdns3_configure_dmult()
2005 if (priv_dev->dev_ver >= DEV_VER_V3 && priv_ep) { in cdns3_configure_dmult()
2008 if (priv_ep->dir) in cdns3_configure_dmult()
2009 mask = BIT(priv_ep->num + 16); in cdns3_configure_dmult()
2011 mask = BIT(priv_ep->num); in cdns3_configure_dmult()
2013 if (priv_ep->type != USB_ENDPOINT_XFER_ISOC && !priv_ep->dir) { in cdns3_configure_dmult()
2014 cdns3_set_register_bit(®s->tdl_from_trb, mask); in cdns3_configure_dmult()
2015 cdns3_set_register_bit(®s->tdl_beh, mask); in cdns3_configure_dmult()
2016 cdns3_set_register_bit(®s->tdl_beh2, mask); in cdns3_configure_dmult()
2017 cdns3_set_register_bit(®s->dma_adv_td, mask); in cdns3_configure_dmult()
2020 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && !priv_ep->dir) in cdns3_configure_dmult()
2021 cdns3_set_register_bit(®s->tdl_from_trb, mask); in cdns3_configure_dmult()
2023 cdns3_set_register_bit(®s->dtrans, mask); in cdns3_configure_dmult()
2034 bool is_iso_ep = (priv_ep->type == USB_ENDPOINT_XFER_ISOC); in cdns3_ep_config()
2035 struct cdns3_device *priv_dev = priv_ep->cdns3_dev; in cdns3_ep_config()
2036 u32 bEndpointAddress = priv_ep->num | priv_ep->dir; in cdns3_ep_config()
2044 buffering = priv_dev->ep_buf_size - 1; in cdns3_ep_config()
2048 switch (priv_ep->type) { in cdns3_ep_config()
2052 if (priv_dev->dev_ver >= DEV_VER_V2 && !priv_ep->dir) in cdns3_ep_config()
2058 if (priv_dev->dev_ver >= DEV_VER_V2 && !priv_ep->dir) in cdns3_ep_config()
2063 mult = priv_dev->ep_iso_burst - 1; in cdns3_ep_config()
2067 switch (priv_dev->gadget.speed) { in cdns3_ep_config()
2078 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC) { in cdns3_ep_config()
2079 maxburst = priv_dev->ep_iso_burst - 1; in cdns3_ep_config()
2083 if (priv_ep->interval > 1) in cdns3_ep_config()
2086 maxburst = priv_dev->ep_buf_size - 1; in cdns3_ep_config()
2091 return -EINVAL; in cdns3_ep_config()
2095 priv_ep->trb_burst_size = 128; in cdns3_ep_config()
2097 priv_ep->trb_burst_size = 64; in cdns3_ep_config()
2099 priv_ep->trb_burst_size = 16; in cdns3_ep_config()
2104 * address is not aligned to 128 Bytes (which is a product of the 64-bit AXI in cdns3_ep_config()
2107 * specifically occurs from the position (4K - (address & 0x7F)) to 4K. in cdns3_ep_config()
2111 if (priv_dev->dev_ver < DEV_VER_V2) in cdns3_ep_config()
2112 priv_ep->trb_burst_size = 16; in cdns3_ep_config()
2119 if (!priv_dev->hw_configured_flag) { in cdns3_ep_config()
2121 !!priv_ep->dir); in cdns3_ep_config()
2123 dev_err(priv_dev->dev, "onchip mem is full, ep is invalid\n"); in cdns3_ep_config()
2131 if (priv_ep->use_streams && priv_dev->gadget.speed >= USB_SPEED_SUPER) { in cdns3_ep_config()
2132 if (priv_dev->dev_ver >= DEV_VER_V3) { in cdns3_ep_config()
2133 u32 mask = BIT(priv_ep->num + (priv_ep->dir ? 16 : 0)); in cdns3_ep_config()
2139 cdns3_clear_register_bit(&priv_dev->regs->tdl_from_trb, in cdns3_ep_config()
2153 writel(ep_cfg, &priv_dev->regs->ep_cfg); in cdns3_ep_config()
2154 priv_ep->flags |= EP_CONFIGURED; in cdns3_ep_config()
2156 dev_dbg(priv_dev->dev, "Configure %s: with val %08x\n", in cdns3_ep_config()
2157 priv_ep->name, ep_cfg); in cdns3_ep_config()
2166 return (priv_ep->endpoint.caps.dir_in && usb_endpoint_dir_in(desc)) || in cdns3_ep_dir_is_correct()
2167 (priv_ep->endpoint.caps.dir_out && usb_endpoint_dir_out(desc)); in cdns3_ep_dir_is_correct()
2177 list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) { in cdns3_find_available_ep()
2181 char c[2] = {ep->name[2], '\0'}; in cdns3_find_available_ep()
2189 if (!(priv_ep->flags & EP_CLAIMED)) { in cdns3_find_available_ep()
2190 priv_ep->num = num; in cdns3_find_available_ep()
2196 return ERR_PTR(-ENOENT); in cdns3_find_available_ep()
2211 * At usb_ep_ops.enable/disable, we only enable and disable endpoint through
2226 dev_err(priv_dev->dev, "no available ep\n"); in cdns3_gadget_match_ep()
2230 dev_dbg(priv_dev->dev, "match endpoint: %s\n", priv_ep->name); in cdns3_gadget_match_ep()
2232 spin_lock_irqsave(&priv_dev->lock, flags); in cdns3_gadget_match_ep()
2233 priv_ep->endpoint.desc = desc; in cdns3_gadget_match_ep()
2234 priv_ep->dir = usb_endpoint_dir_in(desc) ? USB_DIR_IN : USB_DIR_OUT; in cdns3_gadget_match_ep()
2235 priv_ep->type = usb_endpoint_type(desc); in cdns3_gadget_match_ep()
2236 priv_ep->flags |= EP_CLAIMED; in cdns3_gadget_match_ep()
2237 priv_ep->interval = desc->bInterval ? BIT(desc->bInterval - 1) : 0; in cdns3_gadget_match_ep()
2239 spin_unlock_irqrestore(&priv_dev->lock, flags); in cdns3_gadget_match_ep()
2240 return &priv_ep->endpoint; in cdns3_gadget_match_ep()
2260 priv_req->priv_ep = priv_ep; in cdns3_gadget_ep_alloc_request()
2263 return &priv_req->request; in cdns3_gadget_ep_alloc_request()
2276 if (priv_req->aligned_buf) in cdns3_gadget_ep_free_request()
2277 priv_req->aligned_buf->in_use = 0; in cdns3_gadget_ep_free_request()
2305 return -EINVAL; in cdns3_gadget_ep_enable()
2309 priv_dev = priv_ep->cdns3_dev; in cdns3_gadget_ep_enable()
2310 comp_desc = priv_ep->endpoint.comp_desc; in cdns3_gadget_ep_enable()
2312 if (!desc || desc->bDescriptorType != USB_DT_ENDPOINT) { in cdns3_gadget_ep_enable()
2313 dev_dbg(priv_dev->dev, "usbss: invalid parameters\n"); in cdns3_gadget_ep_enable()
2314 return -EINVAL; in cdns3_gadget_ep_enable()
2317 if (!desc->wMaxPacketSize) { in cdns3_gadget_ep_enable()
2318 dev_err(priv_dev->dev, "usbss: missing wMaxPacketSize\n"); in cdns3_gadget_ep_enable()
2319 return -EINVAL; in cdns3_gadget_ep_enable()
2322 if (dev_WARN_ONCE(priv_dev->dev, priv_ep->flags & EP_ENABLED, in cdns3_gadget_ep_enable()
2323 "%s is already enabled\n", priv_ep->name)) in cdns3_gadget_ep_enable()
2326 spin_lock_irqsave(&priv_dev->lock, flags); in cdns3_gadget_ep_enable()
2328 priv_ep->endpoint.desc = desc; in cdns3_gadget_ep_enable()
2329 priv_ep->type = usb_endpoint_type(desc); in cdns3_gadget_ep_enable()
2330 priv_ep->interval = desc->bInterval ? BIT(desc->bInterval - 1) : 0; in cdns3_gadget_ep_enable()
2332 if (priv_ep->interval > ISO_MAX_INTERVAL && in cdns3_gadget_ep_enable()
2333 priv_ep->type == USB_ENDPOINT_XFER_ISOC) { in cdns3_gadget_ep_enable()
2334 dev_err(priv_dev->dev, "Driver is limited to %d period\n", in cdns3_gadget_ep_enable()
2337 ret = -EINVAL; in cdns3_gadget_ep_enable()
2341 bEndpointAddress = priv_ep->num | priv_ep->dir; in cdns3_gadget_ep_enable()
2346 * DMA reads Transfer Ring for the EP which has never got doorbell. in cdns3_gadget_ep_enable()
2352 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && !priv_ep->dir) in cdns3_gadget_ep_enable()
2360 if (priv_dev->gadget.speed >= USB_SPEED_SUPER) { in cdns3_gadget_ep_enable()
2364 priv_ep->use_streams = true; in cdns3_gadget_ep_enable()
2366 priv_dev->using_streams |= true; in cdns3_gadget_ep_enable()
2379 bEndpointAddress = priv_ep->num | priv_ep->dir; in cdns3_gadget_ep_enable()
2384 writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd); in cdns3_gadget_ep_enable()
2386 ret = readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val, in cdns3_gadget_ep_enable()
2392 ret = -EINVAL; in cdns3_gadget_ep_enable()
2397 cdns3_set_register_bit(&priv_dev->regs->ep_ien, in cdns3_gadget_ep_enable()
2400 if (priv_dev->dev_ver < DEV_VER_V2) in cdns3_gadget_ep_enable()
2403 writel(reg, &priv_dev->regs->ep_sts_en); in cdns3_gadget_ep_enable()
2405 ep->desc = desc; in cdns3_gadget_ep_enable()
2406 priv_ep->flags &= ~(EP_PENDING_REQUEST | EP_STALLED | EP_STALL_PENDING | in cdns3_gadget_ep_enable()
2408 priv_ep->flags |= EP_ENABLED | EP_UPDATE_EP_TRBADDR; in cdns3_gadget_ep_enable()
2409 priv_ep->wa1_set = 0; in cdns3_gadget_ep_enable()
2410 priv_ep->enqueue = 0; in cdns3_gadget_ep_enable()
2411 priv_ep->dequeue = 0; in cdns3_gadget_ep_enable()
2412 reg = readl(&priv_dev->regs->ep_sts); in cdns3_gadget_ep_enable()
2413 priv_ep->pcs = !!EP_STS_CCS(reg); in cdns3_gadget_ep_enable()
2414 priv_ep->ccs = !!EP_STS_CCS(reg); in cdns3_gadget_ep_enable()
2416 priv_ep->free_trbs = priv_ep->num_trbs - 1; in cdns3_gadget_ep_enable()
2418 spin_unlock_irqrestore(&priv_dev->lock, flags); in cdns3_gadget_ep_enable()
2424 * cdns3_gadget_ep_disable Disable endpoint
2442 return -EINVAL; in cdns3_gadget_ep_disable()
2446 priv_dev = priv_ep->cdns3_dev; in cdns3_gadget_ep_disable()
2448 if (dev_WARN_ONCE(priv_dev->dev, !(priv_ep->flags & EP_ENABLED), in cdns3_gadget_ep_disable()
2449 "%s is already disabled\n", priv_ep->name)) in cdns3_gadget_ep_disable()
2452 spin_lock_irqsave(&priv_dev->lock, flags); in cdns3_gadget_ep_disable()
2456 cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress); in cdns3_gadget_ep_disable()
2458 ep_cfg = readl(&priv_dev->regs->ep_cfg); in cdns3_gadget_ep_disable()
2460 writel(ep_cfg, &priv_dev->regs->ep_cfg); in cdns3_gadget_ep_disable()
2467 readl_poll_timeout_atomic(&priv_dev->regs->ep_sts, val, in cdns3_gadget_ep_disable()
2469 writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd); in cdns3_gadget_ep_disable()
2471 readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val, in cdns3_gadget_ep_disable()
2475 dev_err(priv_dev->dev, "Timeout: %s resetting failed.\n", in cdns3_gadget_ep_disable()
2476 priv_ep->name); in cdns3_gadget_ep_disable()
2478 while (!list_empty(&priv_ep->pending_req_list)) { in cdns3_gadget_ep_disable()
2479 request = cdns3_next_request(&priv_ep->pending_req_list); in cdns3_gadget_ep_disable()
2482 -ESHUTDOWN); in cdns3_gadget_ep_disable()
2485 while (!list_empty(&priv_ep->wa2_descmiss_req_list)) { in cdns3_gadget_ep_disable()
2486 priv_req = cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list); in cdns3_gadget_ep_disable()
2488 kfree(priv_req->request.buf); in cdns3_gadget_ep_disable()
2489 cdns3_gadget_ep_free_request(&priv_ep->endpoint, in cdns3_gadget_ep_disable()
2490 &priv_req->request); in cdns3_gadget_ep_disable()
2491 list_del_init(&priv_req->list); in cdns3_gadget_ep_disable()
2492 --priv_ep->wa2_counter; in cdns3_gadget_ep_disable()
2495 while (!list_empty(&priv_ep->deferred_req_list)) { in cdns3_gadget_ep_disable()
2496 request = cdns3_next_request(&priv_ep->deferred_req_list); in cdns3_gadget_ep_disable()
2499 -ESHUTDOWN); in cdns3_gadget_ep_disable()
2502 priv_ep->descmis_req = NULL; in cdns3_gadget_ep_disable()
2504 ep->desc = NULL; in cdns3_gadget_ep_disable()
2505 priv_ep->flags &= ~EP_ENABLED; in cdns3_gadget_ep_disable()
2506 priv_ep->use_streams = false; in cdns3_gadget_ep_disable()
2508 spin_unlock_irqrestore(&priv_dev->lock, flags); in cdns3_gadget_ep_disable()
2526 struct cdns3_device *priv_dev = priv_ep->cdns3_dev; in __cdns3_gadget_ep_queue()
2530 request->actual = 0; in __cdns3_gadget_ep_queue()
2531 request->status = -EINPROGRESS; in __cdns3_gadget_ep_queue()
2535 if (priv_dev->dev_ver < DEV_VER_V2) { in __cdns3_gadget_ep_queue()
2547 ret = usb_gadget_map_request_by_dev(priv_dev->sysdev, request, in __cdns3_gadget_ep_queue()
2548 usb_endpoint_dir_in(ep->desc)); in __cdns3_gadget_ep_queue()
2552 list_add_tail(&request->list, &priv_ep->deferred_req_list); in __cdns3_gadget_ep_queue()
2561 if (!request->stream_id) { in __cdns3_gadget_ep_queue()
2562 if (priv_dev->hw_configured_flag && in __cdns3_gadget_ep_queue()
2563 !(priv_ep->flags & EP_STALLED) && in __cdns3_gadget_ep_queue()
2564 !(priv_ep->flags & EP_STALL_PENDING)) in __cdns3_gadget_ep_queue()
2567 if (priv_dev->hw_configured_flag && priv_ep->prime_flag) in __cdns3_gadget_ep_queue()
2584 return -EINVAL; in cdns3_gadget_ep_queue()
2587 priv_dev = priv_ep->cdns3_dev; in cdns3_gadget_ep_queue()
2589 spin_lock_irqsave(&priv_dev->lock, flags); in cdns3_gadget_ep_queue()
2593 if (ret == 0 && request->zero && request->length && in cdns3_gadget_ep_queue()
2594 (request->length % ep->maxpacket == 0)) { in cdns3_gadget_ep_queue()
2598 zlp_request->buf = priv_dev->zlp_buf; in cdns3_gadget_ep_queue()
2599 zlp_request->length = 0; in cdns3_gadget_ep_queue()
2602 priv_req->flags |= REQUEST_ZLP; in cdns3_gadget_ep_queue()
2604 dev_dbg(priv_dev->dev, "Queuing ZLP for endpoint: %s\n", in cdns3_gadget_ep_queue()
2605 priv_ep->name); in cdns3_gadget_ep_queue()
2609 spin_unlock_irqrestore(&priv_dev->lock, flags); in cdns3_gadget_ep_queue()
2632 if (!ep || !request || !ep->desc) in cdns3_gadget_ep_dequeue()
2633 return -EINVAL; in cdns3_gadget_ep_dequeue()
2635 priv_dev = priv_ep->cdns3_dev; in cdns3_gadget_ep_dequeue()
2637 spin_lock_irqsave(&priv_dev->lock, flags); in cdns3_gadget_ep_dequeue()
2643 cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress); in cdns3_gadget_ep_dequeue()
2645 list_for_each_entry_safe(req, req_temp, &priv_ep->pending_req_list, in cdns3_gadget_ep_dequeue()
2653 list_for_each_entry_safe(req, req_temp, &priv_ep->deferred_req_list, in cdns3_gadget_ep_dequeue()
2662 link_trb = priv_req->trb; in cdns3_gadget_ep_dequeue()
2664 /* Update ring only if removed request is on pending_req_list list */ in cdns3_gadget_ep_dequeue()
2666 link_trb->buffer = cpu_to_le32(TRB_BUFFER(priv_ep->trb_pool_dma + in cdns3_gadget_ep_dequeue()
2667 ((priv_req->end_trb + 1) * TRB_SIZE))); in cdns3_gadget_ep_dequeue()
2668 link_trb->control = cpu_to_le32((le32_to_cpu(link_trb->control) & TRB_CYCLE) | in cdns3_gadget_ep_dequeue()
2671 if (priv_ep->wa1_trb == priv_req->trb) in cdns3_gadget_ep_dequeue()
2675 cdns3_gadget_giveback(priv_ep, priv_req, -ECONNRESET); in cdns3_gadget_ep_dequeue()
2678 spin_unlock_irqrestore(&priv_dev->lock, flags); in cdns3_gadget_ep_dequeue()
2689 struct cdns3_device *priv_dev = priv_ep->cdns3_dev; in __cdns3_gadget_ep_set_halt()
2693 if (!(priv_ep->flags & EP_STALLED)) { in __cdns3_gadget_ep_set_halt()
2694 u32 ep_sts_reg = readl(&priv_dev->regs->ep_sts); in __cdns3_gadget_ep_set_halt()
2699 priv_ep->flags |= EP_STALL_PENDING; in __cdns3_gadget_ep_set_halt()
2710 struct cdns3_device *priv_dev = priv_ep->cdns3_dev; in __cdns3_gadget_ep_clear_halt()
2720 request = cdns3_next_request(&priv_ep->pending_req_list); in __cdns3_gadget_ep_clear_halt()
2723 trb = priv_req->trb; in __cdns3_gadget_ep_clear_halt()
2726 trb->control = trb->control ^ cpu_to_le32(TRB_CYCLE); in __cdns3_gadget_ep_clear_halt()
2730 writel(EP_CMD_CSTALL | EP_CMD_EPRST, &priv_dev->regs->ep_cmd); in __cdns3_gadget_ep_clear_halt()
2733 ret = readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val, in __cdns3_gadget_ep_clear_halt()
2736 return -EINVAL; in __cdns3_gadget_ep_clear_halt()
2738 priv_ep->flags &= ~(EP_STALLED | EP_STALL_PENDING); in __cdns3_gadget_ep_clear_halt()
2761 struct cdns3_device *priv_dev = priv_ep->cdns3_dev; in cdns3_gadget_ep_set_halt()
2765 if (!(priv_ep->flags & EP_ENABLED)) in cdns3_gadget_ep_set_halt()
2766 return -EPERM; in cdns3_gadget_ep_set_halt()
2768 spin_lock_irqsave(&priv_dev->lock, flags); in cdns3_gadget_ep_set_halt()
2770 cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress); in cdns3_gadget_ep_set_halt()
2773 priv_ep->flags &= ~EP_WEDGE; in cdns3_gadget_ep_set_halt()
2779 spin_unlock_irqrestore(&priv_dev->lock, flags); in cdns3_gadget_ep_set_halt()
2788 .disable = cdns3_gadget_ep_disable,
2807 return readl(&priv_dev->regs->usb_itpn); in cdns3_gadget_get_frame()
2820 writel(USB_CONF_LGO_L0, &priv_dev->regs->usb_conf); in __cdns3_gadget_wakeup()
2831 spin_lock_irqsave(&priv_dev->lock, flags); in cdns3_gadget_wakeup()
2833 spin_unlock_irqrestore(&priv_dev->lock, flags); in cdns3_gadget_wakeup()
2843 spin_lock_irqsave(&priv_dev->lock, flags); in cdns3_gadget_set_selfpowered()
2844 priv_dev->is_selfpowered = !!is_selfpowered; in cdns3_gadget_set_selfpowered()
2845 spin_unlock_irqrestore(&priv_dev->lock, flags); in cdns3_gadget_set_selfpowered()
2854 writel(USB_CONF_DEVEN, &priv_dev->regs->usb_conf); in cdns3_gadget_pullup()
2856 writel(~0, &priv_dev->regs->ep_ists); in cdns3_gadget_pullup()
2857 writel(~0, &priv_dev->regs->usb_ists); in cdns3_gadget_pullup()
2858 writel(USB_CONF_DEVDS, &priv_dev->regs->usb_conf); in cdns3_gadget_pullup()
2866 struct cdns3_usb_regs __iomem *regs = priv_dev->regs; in cdns3_gadget_config()
2872 writel(EP_IEN_EP_OUT0 | EP_IEN_EP_IN0, ®s->ep_ien); in cdns3_gadget_config()
2878 if (priv_dev->dev_ver == DEV_VER_TI_V1) { in cdns3_gadget_config()
2879 reg = readl(®s->dbg_link1); in cdns3_gadget_config()
2884 writel(reg, ®s->dbg_link1); in cdns3_gadget_config()
2889 * This cause problem with cache, so driver restore non-secure in cdns3_gadget_config()
2892 reg = readl(®s->dma_axi_ctrl); in cdns3_gadget_config()
2895 writel(reg, ®s->dma_axi_ctrl); in cdns3_gadget_config()
2898 writel(USB_IEN_INIT, ®s->usb_ien); in cdns3_gadget_config()
2899 writel(USB_CONF_CLK2OFFDS | USB_CONF_L1DS, ®s->usb_conf); in cdns3_gadget_config()
2901 writel(PUSB_PWR_FST_REG_ACCESS, &priv_dev->regs->usb_pwr); in cdns3_gadget_config()
2918 enum usb_device_speed max_speed = driver->max_speed; in cdns3_gadget_udc_start()
2920 spin_lock_irqsave(&priv_dev->lock, flags); in cdns3_gadget_udc_start()
2921 priv_dev->gadget_driver = driver; in cdns3_gadget_udc_start()
2924 max_speed = min(driver->max_speed, gadget->max_speed); in cdns3_gadget_udc_start()
2928 writel(USB_CONF_SFORCE_FS, &priv_dev->regs->usb_conf); in cdns3_gadget_udc_start()
2929 writel(USB_CONF_USB3DIS, &priv_dev->regs->usb_conf); in cdns3_gadget_udc_start()
2932 writel(USB_CONF_USB3DIS, &priv_dev->regs->usb_conf); in cdns3_gadget_udc_start()
2937 dev_err(priv_dev->dev, in cdns3_gadget_udc_start()
2948 spin_unlock_irqrestore(&priv_dev->lock, flags); in cdns3_gadget_udc_start()
2966 priv_dev->gadget_driver = NULL; in cdns3_gadget_udc_stop()
2968 priv_dev->onchip_used_size = 0; in cdns3_gadget_udc_stop()
2969 priv_dev->out_mem_is_allocated = 0; in cdns3_gadget_udc_stop()
2970 priv_dev->gadget.speed = USB_SPEED_UNKNOWN; in cdns3_gadget_udc_stop()
2972 list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) { in cdns3_gadget_udc_stop()
2974 bEndpointAddress = priv_ep->num | priv_ep->dir; in cdns3_gadget_udc_stop()
2976 writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd); in cdns3_gadget_udc_stop()
2977 readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val, in cdns3_gadget_udc_stop()
2980 priv_ep->flags &= ~EP_CLAIMED; in cdns3_gadget_udc_stop()
2983 /* disable interrupt for device */ in cdns3_gadget_udc_stop()
2984 writel(0, &priv_dev->regs->usb_ien); in cdns3_gadget_udc_stop()
2985 writel(0, &priv_dev->regs->usb_pwr); in cdns3_gadget_udc_stop()
2986 writel(USB_CONF_DEVDS, &priv_dev->regs->usb_conf); in cdns3_gadget_udc_stop()
2992 * cdns3_gadget_check_config - ensure cdns3 can support the USB configuration
3008 list_for_each_entry(ep, &gadget->ep_list, ep_list) { in cdns3_gadget_check_config()
3009 if (ep->claimed && (ep->address & USB_DIR_IN)) in cdns3_gadget_check_config()
3016 if (total > priv_dev->onchip_buffers) in cdns3_gadget_check_config()
3017 return -ENOMEM; in cdns3_gadget_check_config()
3019 priv_dev->ep_buf_size = priv_dev->ep_iso_burst = in cdns3_gadget_check_config()
3020 (priv_dev->onchip_buffers - 2) / (n_in + 1); in cdns3_gadget_check_config()
3029 .pullup = cdns3_gadget_pullup,
3041 priv_dev->eps[16] = NULL; in cdns3_free_all_eps()
3044 if (priv_dev->eps[i]) { in cdns3_free_all_eps()
3045 cdns3_free_trb_pool(priv_dev->eps[i]); in cdns3_free_all_eps()
3046 devm_kfree(priv_dev->dev, priv_dev->eps[i]); in cdns3_free_all_eps()
3066 ep_enabled_reg = readl(&priv_dev->regs->usb_cap3); in cdns3_init_eps()
3067 iso_ep_reg = readl(&priv_dev->regs->usb_cap4); in cdns3_init_eps()
3069 dev_dbg(priv_dev->dev, "Initializing non-zero endpoints\n"); in cdns3_init_eps()
3080 priv_dev->eps[i] = priv_dev->eps[0]; in cdns3_init_eps()
3084 priv_ep = devm_kzalloc(priv_dev->dev, sizeof(*priv_ep), in cdns3_init_eps()
3090 priv_ep->cdns3_dev = priv_dev; in cdns3_init_eps()
3091 priv_dev->eps[i] = priv_ep; in cdns3_init_eps()
3092 priv_ep->num = ep_number; in cdns3_init_eps()
3093 priv_ep->dir = ep_dir ? USB_DIR_IN : USB_DIR_OUT; in cdns3_init_eps()
3098 dev_err(priv_dev->dev, "Failed to init ep0\n"); in cdns3_init_eps()
3102 snprintf(priv_ep->name, sizeof(priv_ep->name), "ep%d%s", in cdns3_init_eps()
3104 priv_ep->endpoint.name = priv_ep->name; in cdns3_init_eps()
3106 usb_ep_set_maxpacket_limit(&priv_ep->endpoint, in cdns3_init_eps()
3108 priv_ep->endpoint.max_streams = CDNS3_EP_MAX_STREAMS; in cdns3_init_eps()
3109 priv_ep->endpoint.ops = &cdns3_gadget_ep_ops; in cdns3_init_eps()
3111 priv_ep->endpoint.caps.dir_in = 1; in cdns3_init_eps()
3113 priv_ep->endpoint.caps.dir_out = 1; in cdns3_init_eps()
3116 priv_ep->endpoint.caps.type_iso = 1; in cdns3_init_eps()
3118 priv_ep->endpoint.caps.type_bulk = 1; in cdns3_init_eps()
3119 priv_ep->endpoint.caps.type_int = 1; in cdns3_init_eps()
3121 list_add_tail(&priv_ep->endpoint.ep_list, in cdns3_init_eps()
3122 &priv_dev->gadget.ep_list); in cdns3_init_eps()
3125 priv_ep->flags = 0; in cdns3_init_eps()
3127 dev_dbg(priv_dev->dev, "Initialized %s support: %s %s\n", in cdns3_init_eps()
3128 priv_ep->name, in cdns3_init_eps()
3129 priv_ep->endpoint.caps.type_bulk ? "BULK, INT" : "", in cdns3_init_eps()
3130 priv_ep->endpoint.caps.type_iso ? "ISO" : ""); in cdns3_init_eps()
3132 INIT_LIST_HEAD(&priv_ep->pending_req_list); in cdns3_init_eps()
3133 INIT_LIST_HEAD(&priv_ep->deferred_req_list); in cdns3_init_eps()
3134 INIT_LIST_HEAD(&priv_ep->wa2_descmiss_req_list); in cdns3_init_eps()
3140 return -ENOMEM; in cdns3_init_eps()
3155 priv_dev = cdns->gadget_dev; in cdns3_gadget_exit()
3158 pm_runtime_mark_last_busy(cdns->dev); in cdns3_gadget_exit()
3159 pm_runtime_put_autosuspend(cdns->dev); in cdns3_gadget_exit()
3161 usb_del_gadget(&priv_dev->gadget); in cdns3_gadget_exit()
3162 devm_free_irq(cdns->dev, cdns->dev_irq, priv_dev); in cdns3_gadget_exit()
3166 while (!list_empty(&priv_dev->aligned_buf_list)) { in cdns3_gadget_exit()
3169 buf = cdns3_next_align_buf(&priv_dev->aligned_buf_list); in cdns3_gadget_exit()
3170 dma_free_coherent(priv_dev->sysdev, buf->size, in cdns3_gadget_exit()
3171 buf->buf, in cdns3_gadget_exit()
3172 buf->dma); in cdns3_gadget_exit()
3174 list_del(&buf->list); in cdns3_gadget_exit()
3178 dma_free_coherent(priv_dev->sysdev, 8, priv_dev->setup_buf, in cdns3_gadget_exit()
3179 priv_dev->setup_dma); in cdns3_gadget_exit()
3181 kfree(priv_dev->zlp_buf); in cdns3_gadget_exit()
3182 usb_put_gadget(&priv_dev->gadget); in cdns3_gadget_exit()
3183 cdns->gadget_dev = NULL; in cdns3_gadget_exit()
3195 return -ENOMEM; in cdns3_gadget_start()
3197 usb_initialize_gadget(cdns->dev, &priv_dev->gadget, in cdns3_gadget_start()
3199 cdns->gadget_dev = priv_dev; in cdns3_gadget_start()
3200 priv_dev->sysdev = cdns->dev; in cdns3_gadget_start()
3201 priv_dev->dev = cdns->dev; in cdns3_gadget_start()
3202 priv_dev->regs = cdns->dev_regs; in cdns3_gadget_start()
3204 device_property_read_u16(priv_dev->dev, "cdns,on-chip-buff-size", in cdns3_gadget_start()
3205 &priv_dev->onchip_buffers); in cdns3_gadget_start()
3207 if (priv_dev->onchip_buffers <= 0) { in cdns3_gadget_start()
3208 u32 reg = readl(&priv_dev->regs->usb_cap2); in cdns3_gadget_start()
3210 priv_dev->onchip_buffers = USB_CAP2_ACTUAL_MEM_SIZE(reg); in cdns3_gadget_start()
3213 if (!priv_dev->onchip_buffers) in cdns3_gadget_start()
3214 priv_dev->onchip_buffers = 256; in cdns3_gadget_start()
3216 max_speed = usb_get_maximum_speed(cdns->dev); in cdns3_gadget_start()
3225 dev_err(cdns->dev, "invalid maximum_speed parameter %d\n", in cdns3_gadget_start()
3235 priv_dev->gadget.max_speed = max_speed; in cdns3_gadget_start()
3236 priv_dev->gadget.speed = USB_SPEED_UNKNOWN; in cdns3_gadget_start()
3237 priv_dev->gadget.ops = &cdns3_gadget_ops; in cdns3_gadget_start()
3238 priv_dev->gadget.name = "usb-ss-gadget"; in cdns3_gadget_start()
3239 priv_dev->gadget.quirk_avoids_skb_reserve = 1; in cdns3_gadget_start()
3240 priv_dev->gadget.irq = cdns->dev_irq; in cdns3_gadget_start()
3242 spin_lock_init(&priv_dev->lock); in cdns3_gadget_start()
3243 INIT_WORK(&priv_dev->pending_status_wq, in cdns3_gadget_start()
3246 INIT_WORK(&priv_dev->aligned_buf_wq, in cdns3_gadget_start()
3250 INIT_LIST_HEAD(&priv_dev->gadget.ep_list); in cdns3_gadget_start()
3251 INIT_LIST_HEAD(&priv_dev->aligned_buf_list); in cdns3_gadget_start()
3255 dev_err(priv_dev->dev, "Failed to create endpoints\n"); in cdns3_gadget_start()
3260 priv_dev->setup_buf = dma_alloc_coherent(priv_dev->sysdev, 8, in cdns3_gadget_start()
3261 &priv_dev->setup_dma, GFP_DMA); in cdns3_gadget_start()
3262 if (!priv_dev->setup_buf) { in cdns3_gadget_start()
3263 ret = -ENOMEM; in cdns3_gadget_start()
3267 priv_dev->dev_ver = readl(&priv_dev->regs->usb_cap6); in cdns3_gadget_start()
3269 dev_dbg(priv_dev->dev, "Device Controller version: %08x\n", in cdns3_gadget_start()
3270 readl(&priv_dev->regs->usb_cap6)); in cdns3_gadget_start()
3271 dev_dbg(priv_dev->dev, "USB Capabilities:: %08x\n", in cdns3_gadget_start()
3272 readl(&priv_dev->regs->usb_cap1)); in cdns3_gadget_start()
3273 dev_dbg(priv_dev->dev, "On-Chip memory configuration: %08x\n", in cdns3_gadget_start()
3274 readl(&priv_dev->regs->usb_cap2)); in cdns3_gadget_start()
3276 priv_dev->dev_ver = GET_DEV_BASE_VERSION(priv_dev->dev_ver); in cdns3_gadget_start()
3277 if (priv_dev->dev_ver >= DEV_VER_V2) in cdns3_gadget_start()
3278 priv_dev->gadget.sg_supported = 1; in cdns3_gadget_start()
3280 priv_dev->zlp_buf = kzalloc(CDNS3_EP_ZLP_BUF_SIZE, GFP_KERNEL); in cdns3_gadget_start()
3281 if (!priv_dev->zlp_buf) { in cdns3_gadget_start()
3282 ret = -ENOMEM; in cdns3_gadget_start()
3287 ret = usb_add_gadget(&priv_dev->gadget); in cdns3_gadget_start()
3289 dev_err(priv_dev->dev, "Failed to add gadget\n"); in cdns3_gadget_start()
3295 kfree(priv_dev->zlp_buf); in cdns3_gadget_start()
3297 dma_free_coherent(priv_dev->sysdev, 8, priv_dev->setup_buf, in cdns3_gadget_start()
3298 priv_dev->setup_dma); in cdns3_gadget_start()
3302 usb_put_gadget(&priv_dev->gadget); in cdns3_gadget_start()
3303 cdns->gadget_dev = NULL; in cdns3_gadget_start()
3311 /* Ensure 32-bit DMA Mask in case we switched back from Host mode */ in __cdns3_gadget_init()
3312 ret = dma_set_mask_and_coherent(cdns->dev, DMA_BIT_MASK(32)); in __cdns3_gadget_init()
3314 dev_err(cdns->dev, "Failed to set dma mask: %d\n", ret); in __cdns3_gadget_init()
3319 pm_runtime_get_sync(cdns->dev); in __cdns3_gadget_init()
3323 pm_runtime_put_sync(cdns->dev); in __cdns3_gadget_init()
3331 ret = devm_request_threaded_irq(cdns->dev, cdns->dev_irq, in __cdns3_gadget_init()
3334 IRQF_SHARED, dev_name(cdns->dev), in __cdns3_gadget_init()
3335 cdns->gadget_dev); in __cdns3_gadget_init()
3347 __must_hold(&cdns->lock) in cdns3_gadget_suspend()
3349 struct cdns3_device *priv_dev = cdns->gadget_dev; in cdns3_gadget_suspend()
3351 spin_unlock(&cdns->lock); in cdns3_gadget_suspend()
3353 spin_lock(&cdns->lock); in cdns3_gadget_suspend()
3355 priv_dev->gadget.speed = USB_SPEED_UNKNOWN; in cdns3_gadget_suspend()
3356 usb_gadget_set_state(&priv_dev->gadget, USB_STATE_NOTATTACHED); in cdns3_gadget_suspend()
3359 /* disable interrupt for device */ in cdns3_gadget_suspend()
3360 writel(0, &priv_dev->regs->usb_ien); in cdns3_gadget_suspend()
3367 struct cdns3_device *priv_dev = cdns->gadget_dev; in cdns3_gadget_resume()
3369 if (!priv_dev->gadget_driver) in cdns3_gadget_resume()
3378 * cdns3_gadget_init - initialize device structure
3388 rdrv = devm_kzalloc(cdns->dev, sizeof(*rdrv), GFP_KERNEL); in cdns3_gadget_init()
3390 return -ENOMEM; in cdns3_gadget_init()
3392 rdrv->start = __cdns3_gadget_init; in cdns3_gadget_init()
3393 rdrv->stop = cdns3_gadget_exit; in cdns3_gadget_init()
3394 rdrv->suspend = cdns3_gadget_suspend; in cdns3_gadget_init()
3395 rdrv->resume = cdns3_gadget_resume; in cdns3_gadget_init()
3396 rdrv->state = CDNS3_ROLE_STATE_INACTIVE; in cdns3_gadget_init()
3397 rdrv->name = "gadget"; in cdns3_gadget_init()
3398 cdns->roles[USB_ROLE_DEVICE] = rdrv; in cdns3_gadget_init()