337 static inline u32 trb_read_##name(struct tegra_xudc_trb *trb)		\
339 	return (le32_to_cpu(trb->member) >> (shift)) & (mask);		\
342 trb_write_##name(struct tegra_xudc_trb *trb, u32 val)			\
346 	tmp = le32_to_cpu(trb->member) & ~((mask) << (shift));		\
348 	trb->member = cpu_to_le32(tmp);					\
370 static inline u64 trb_read_data_ptr(struct tegra_xudc_trb *trb)  in trb_read_data_ptr()  argument
372 	return ((u64)trb_read_data_hi(trb) << 32) |  in trb_read_data_ptr()
373 		trb_read_data_lo(trb);  in trb_read_data_ptr()
376 static inline void trb_write_data_ptr(struct tegra_xudc_trb *trb, u64 addr)  in trb_write_data_ptr()  argument
378 	trb_write_data_lo(trb, lower_32_bits(addr));  in trb_write_data_ptr()
379 	trb_write_data_hi(trb, upper_32_bits(addr));  in trb_write_data_ptr()
619 			    struct tegra_xudc_trb *trb)  in dump_trb()  argument
623 		type, trb, trb->data_lo, trb->data_hi, trb->status,  in dump_trb()
624 		trb->control);  in dump_trb()
886 				   struct tegra_xudc_trb *trb)  in trb_virt_to_phys()  argument
890 	index = trb - ep->transfer_ring;  in trb_virt_to_phys()
895 	return (ep->transfer_ring_phys + index * sizeof(*trb));  in trb_virt_to_phys()
901 	struct tegra_xudc_trb *trb;  in trb_phys_to_virt()  local
904 	index = (addr - ep->transfer_ring_phys) / sizeof(*trb);  in trb_phys_to_virt()
909 	trb = &ep->transfer_ring[index];  in trb_phys_to_virt()
911 	return trb;  in trb_phys_to_virt()
1071 				     struct tegra_xudc_trb *trb,  in tegra_xudc_queue_one_trb()  argument
1085 	trb_write_data_ptr(trb, buf_addr);  in tegra_xudc_queue_one_trb()
1087 	trb_write_transfer_len(trb, len);  in tegra_xudc_queue_one_trb()
1088 	trb_write_td_size(trb, req->trbs_needed - req->trbs_queued - 1);  in tegra_xudc_queue_one_trb()
1092 		trb_write_chain(trb, 0);  in tegra_xudc_queue_one_trb()
1094 		trb_write_chain(trb, 1);  in tegra_xudc_queue_one_trb()
1096 	trb_write_ioc(trb, ioc);  in tegra_xudc_queue_one_trb()
1101 		trb_write_isp(trb, 1);  in tegra_xudc_queue_one_trb()
1103 		trb_write_isp(trb, 0);  in tegra_xudc_queue_one_trb()
1108 			trb_write_type(trb, TRB_TYPE_DATA_STAGE);  in tegra_xudc_queue_one_trb()
1110 			trb_write_type(trb, TRB_TYPE_STATUS_STAGE);  in tegra_xudc_queue_one_trb()
1114 			trb_write_data_stage_dir(trb, 1);  in tegra_xudc_queue_one_trb()
1116 			trb_write_data_stage_dir(trb, 0);  in tegra_xudc_queue_one_trb()
1118 		trb_write_type(trb, TRB_TYPE_ISOCH);  in tegra_xudc_queue_one_trb()
1119 		trb_write_sia(trb, 1);  in tegra_xudc_queue_one_trb()
1120 		trb_write_frame_id(trb, 0);  in tegra_xudc_queue_one_trb()
1121 		trb_write_tlbpc(trb, 0);  in tegra_xudc_queue_one_trb()
1123 		trb_write_type(trb, TRB_TYPE_STREAM);  in tegra_xudc_queue_one_trb()
1124 		trb_write_stream_id(trb, req->usb_req.stream_id);  in tegra_xudc_queue_one_trb()
1126 		trb_write_type(trb, TRB_TYPE_NORMAL);  in tegra_xudc_queue_one_trb()
1127 		trb_write_stream_id(trb, 0);  in tegra_xudc_queue_one_trb()
1130 	trb_write_cycle(trb, ep->pcs);  in tegra_xudc_queue_one_trb()
1135 	dump_trb(xudc, "TRANSFER", trb);  in tegra_xudc_queue_one_trb()
1175 		struct tegra_xudc_trb *trb = &ep->transfer_ring[ep->enq_ptr];  in tegra_xudc_queue_trbs()  local
1181 		tegra_xudc_queue_one_trb(ep, req, trb, ioc);  in tegra_xudc_queue_trbs()
1182 		req->last_trb = trb;  in tegra_xudc_queue_trbs()
1186 			trb = &ep->transfer_ring[ep->enq_ptr];  in tegra_xudc_queue_trbs()
1187 			trb_write_cycle(trb, ep->pcs);  in tegra_xudc_queue_trbs()
1328 	struct tegra_xudc_trb *trb = req->first_trb;  in squeeze_transfer_ring()  local
1329 	bool pcs_enq = trb_read_cycle(trb);  in squeeze_transfer_ring()
1336 	while (trb != &ep->transfer_ring[ep->enq_ptr]) {  in squeeze_transfer_ring()
1337 		pcs = trb_read_cycle(trb);  in squeeze_transfer_ring()
1338 		memset(trb, 0, sizeof(*trb));  in squeeze_transfer_ring()
1339 		trb_write_cycle(trb, !pcs);  in squeeze_transfer_ring()
1340 		trb++;  in squeeze_transfer_ring()
1342 		if (trb_read_type(trb) == TRB_TYPE_LINK)  in squeeze_transfer_ring()
1343 			trb = ep->transfer_ring;  in squeeze_transfer_ring()
1371 			   struct tegra_xudc_trb *trb)  in trb_in_request()  argument
1374 		req->first_trb, req->last_trb, trb);  in trb_in_request()
1376 	if (trb >= req->first_trb && (trb <= req->last_trb ||  in trb_in_request()
1380 	if (trb < req->first_trb && trb <= req->last_trb &&  in trb_in_request()
1393 			       struct tegra_xudc_trb *trb)  in trb_before_request()  argument
1398 		__func__, req->first_trb, req->last_trb, enq_trb, trb);  in trb_before_request()
1400 	if (trb < req->first_trb && (enq_trb <= trb ||  in trb_before_request()
1404 	if (trb > req->first_trb && req->first_trb < enq_trb && enq_trb <= trb)  in trb_before_request()
1689 			   struct tegra_xudc_trb *trb)  in setup_link_trb()  argument
1691 	trb_write_data_ptr(trb, ep->transfer_ring_phys);  in setup_link_trb()
1692 	trb_write_type(trb, TRB_TYPE_LINK);  in setup_link_trb()
1693 	trb_write_toggle_cycle(trb, 1);  in setup_link_trb()
2624 trb_to_request(struct tegra_xudc_ep *ep, struct tegra_xudc_trb *trb)  in trb_to_request()  argument
2632 		if (trb_in_request(ep, req, trb))  in trb_to_request()
2644 	struct tegra_xudc_trb *trb;  in tegra_xudc_handle_transfer_completion()  local
2650 	trb = trb_phys_to_virt(ep, trb_read_data_ptr(event));  in tegra_xudc_handle_transfer_completion()
2651 	req = trb_to_request(ep, trb);  in tegra_xudc_handle_transfer_completion()
2657 	if (req && (short_packet || (!trb_read_chain(trb) &&  in tegra_xudc_handle_transfer_completion()
2695 	struct tegra_xudc_trb *trb;  in tegra_xudc_handle_transfer_event()  local
2705 	trb = trb_phys_to_virt(ep, trb_read_data_ptr(event));  in tegra_xudc_handle_transfer_event()
2708 		ep->deq_ptr = (trb - ep->transfer_ring) + 1;  in tegra_xudc_handle_transfer_event()