97 static void ioat3_eh(struct ioat2_dma_chan *ioat);
299 static void ioat3_dma_unmap(struct ioat2_dma_chan *ioat,  in ioat3_dma_unmap()  argument
302 	struct ioat_chan_common *chan = &ioat->base;  in ioat3_dma_unmap()
332 			ext = ioat2_get_ring_ent(ioat, idx + 1);  in ioat3_dma_unmap()
369 			ext = ioat2_get_ring_ent(ioat, idx + 1);  in ioat3_dma_unmap()
514 desc_get_errstat(struct ioat2_dma_chan *ioat, struct ioat_ring_ent *desc)  in desc_get_errstat()  argument
550 static void __cleanup(struct ioat2_dma_chan *ioat, dma_addr_t phys_complete)  in __cleanup()  argument
552 	struct ioat_chan_common *chan = &ioat->base;  in __cleanup()
556 	int idx = ioat->tail, i;  in __cleanup()
560 		__func__, ioat->head, ioat->tail, ioat->issued);  in __cleanup()
572 	active = ioat2_ring_active(ioat);  in __cleanup()
577 		prefetch(ioat2_get_ring_ent(ioat, idx + i + 1));  in __cleanup()
578 		desc = ioat2_get_ring_ent(ioat, idx + i);  in __cleanup()
579 		dump_desc_dbg(ioat, desc);  in __cleanup()
583 			desc_get_errstat(ioat, desc);  in __cleanup()
588 			ioat3_dma_unmap(ioat, desc, idx + i);  in __cleanup()
611 	ioat->tail = idx + i;  in __cleanup()
626 static void ioat3_cleanup(struct ioat2_dma_chan *ioat)  in ioat3_cleanup()  argument
628 	struct ioat_chan_common *chan = &ioat->base;  in ioat3_cleanup()
634 		__cleanup(ioat, phys_complete);  in ioat3_cleanup()
641 			ioat3_eh(ioat);  in ioat3_cleanup()
650 	struct ioat2_dma_chan *ioat = to_ioat2_chan((void *) data);  in ioat3_cleanup_event()  local
652 	ioat3_cleanup(ioat);  in ioat3_cleanup_event()
653 	writew(IOAT_CHANCTRL_RUN, ioat->base.reg_base + IOAT_CHANCTRL_OFFSET);  in ioat3_cleanup_event()
656 static void ioat3_restart_channel(struct ioat2_dma_chan *ioat)  in ioat3_restart_channel()  argument
658 	struct ioat_chan_common *chan = &ioat->base;  in ioat3_restart_channel()
663 		__cleanup(ioat, phys_complete);  in ioat3_restart_channel()
665 	__ioat2_restart_chan(ioat);  in ioat3_restart_channel()
668 static void ioat3_eh(struct ioat2_dma_chan *ioat)  in ioat3_eh()  argument
670 	struct ioat_chan_common *chan = &ioat->base;  in ioat3_eh()
681 		__cleanup(ioat, phys_complete);  in ioat3_eh()
689 	desc = ioat2_get_ring_ent(ioat, ioat->tail);  in ioat3_eh()
691 	dump_desc_dbg(ioat, desc);  in ioat3_eh()
726 	spin_lock_bh(&ioat->prep_lock);  in ioat3_eh()
727 	ioat3_restart_channel(ioat);  in ioat3_eh()
728 	spin_unlock_bh(&ioat->prep_lock);  in ioat3_eh()
731 static void check_active(struct ioat2_dma_chan *ioat)  in check_active()  argument
733 	struct ioat_chan_common *chan = &ioat->base;  in check_active()
735 	if (ioat2_ring_active(ioat)) {  in check_active()
742 	else if (ioat->alloc_order > ioat_get_alloc_order()) {  in check_active()
746 		reshape_ring(ioat, ioat->alloc_order - 1);  in check_active()
751 		if (ioat->alloc_order > ioat_get_alloc_order())  in check_active()
759 	struct ioat2_dma_chan *ioat = to_ioat2_chan((void *) data);  in ioat3_timer_event()  local
760 	struct ioat_chan_common *chan = &ioat->base;  in ioat3_timer_event()
787 		__cleanup(ioat, phys_complete);  in ioat3_timer_event()
789 		spin_lock_bh(&ioat->prep_lock);  in ioat3_timer_event()
790 		ioat3_restart_channel(ioat);  in ioat3_timer_event()
791 		spin_unlock_bh(&ioat->prep_lock);  in ioat3_timer_event()
800 	if (ioat2_ring_active(ioat))  in ioat3_timer_event()
803 		spin_lock_bh(&ioat->prep_lock);  in ioat3_timer_event()
804 		check_active(ioat);  in ioat3_timer_event()
805 		spin_unlock_bh(&ioat->prep_lock);  in ioat3_timer_event()
814 	struct ioat2_dma_chan *ioat = to_ioat2_chan(c);  in ioat3_tx_status()  local
821 	ioat3_cleanup(ioat);  in ioat3_tx_status()
830 	struct ioat2_dma_chan *ioat = to_ioat2_chan(c);  in ioat3_prep_memset_lock()  local
837 	num_descs = ioat2_xferlen_to_descs(ioat, len);  in ioat3_prep_memset_lock()
838 	if (likely(num_descs) && ioat2_check_space_lock(ioat, num_descs) == 0)  in ioat3_prep_memset_lock()
839 		idx = ioat->head;  in ioat3_prep_memset_lock()
844 		size_t xfer_size = min_t(size_t, len, 1 << ioat->xfercap_log);  in ioat3_prep_memset_lock()
846 		desc = ioat2_get_ring_ent(ioat, idx + i);  in ioat3_prep_memset_lock()
857 		dump_desc_dbg(ioat, desc);  in ioat3_prep_memset_lock()
865 	dump_desc_dbg(ioat, desc);  in ioat3_prep_memset_lock()
876 	struct ioat2_dma_chan *ioat = to_ioat2_chan(c);  in __ioat3_prep_xor_lock()  local
890 	num_descs = ioat2_xferlen_to_descs(ioat, len);  in __ioat3_prep_xor_lock()
905 	if (likely(num_descs) && ioat2_check_space_lock(ioat, num_descs+1) == 0)  in __ioat3_prep_xor_lock()
906 		idx = ioat->head;  in __ioat3_prep_xor_lock()
912 		size_t xfer_size = min_t(size_t, len, 1 << ioat->xfercap_log);  in __ioat3_prep_xor_lock()
915 		desc = ioat2_get_ring_ent(ioat, idx + i);  in __ioat3_prep_xor_lock()
922 		ext = ioat2_get_ring_ent(ioat, idx + i + 1);  in __ioat3_prep_xor_lock()
937 		dump_desc_dbg(ioat, desc);  in __ioat3_prep_xor_lock()
948 	compl_desc = ioat2_get_ring_ent(ioat, idx + i);  in __ioat3_prep_xor_lock()
956 	dump_desc_dbg(ioat, compl_desc);  in __ioat3_prep_xor_lock()
984 dump_pq_desc_dbg(struct ioat2_dma_chan *ioat, struct ioat_ring_ent *desc, struct ioat_ring_ent *ext)  in dump_pq_desc_dbg()  argument
986 	struct device *dev = to_dev(&ioat->base);  in dump_pq_desc_dbg()
1010 static void dump_pq16_desc_dbg(struct ioat2_dma_chan *ioat,  in dump_pq16_desc_dbg()  argument
1013 	struct device *dev = to_dev(&ioat->base);  in dump_pq16_desc_dbg()
1051 	struct ioat2_dma_chan *ioat = to_ioat2_chan(c);  in __ioat3_prep_pq_lock()  local
1052 	struct ioat_chan_common *chan = &ioat->base;  in __ioat3_prep_pq_lock()
1072 	num_descs = ioat2_xferlen_to_descs(ioat, len);  in __ioat3_prep_pq_lock()
1090 	    ioat2_check_space_lock(ioat, num_descs + cb32) == 0)  in __ioat3_prep_pq_lock()
1091 		idx = ioat->head;  in __ioat3_prep_pq_lock()
1097 		size_t xfer_size = min_t(size_t, len, 1 << ioat->xfercap_log);  in __ioat3_prep_pq_lock()
1099 		desc = ioat2_get_ring_ent(ioat, idx + i);  in __ioat3_prep_pq_lock()
1106 		ext = ioat2_get_ring_ent(ioat, idx + i + with_ext);  in __ioat3_prep_pq_lock()
1145 	dump_pq_desc_dbg(ioat, desc, ext);  in __ioat3_prep_pq_lock()
1153 		compl_desc = ioat2_get_ring_ent(ioat, idx + i);  in __ioat3_prep_pq_lock()
1161 		dump_desc_dbg(ioat, compl_desc);  in __ioat3_prep_pq_lock()
1175 	struct ioat2_dma_chan *ioat = to_ioat2_chan(c);  in __ioat3_prep_pq16_lock()  local
1176 	struct ioat_chan_common *chan = &ioat->base;  in __ioat3_prep_pq16_lock()
1193 	num_descs = ioat2_xferlen_to_descs(ioat, len);  in __ioat3_prep_pq16_lock()
1199 	if (num_descs && ioat2_check_space_lock(ioat, num_descs) == 0)  in __ioat3_prep_pq16_lock()
1200 		idx = ioat->head;  in __ioat3_prep_pq16_lock()
1208 		size_t xfer_size = min_t(size_t, len, 1 << ioat->xfercap_log);  in __ioat3_prep_pq16_lock()
1210 		desc = ioat2_get_ring_ent(ioat, idx + i);  in __ioat3_prep_pq16_lock()
1268 	dump_pq16_desc_dbg(ioat, desc);  in __ioat3_prep_pq16_lock()
1391 	struct ioat2_dma_chan *ioat = to_ioat2_chan(c);  in ioat3_prep_interrupt_lock()  local
1395 	if (ioat2_check_space_lock(ioat, 1) == 0)  in ioat3_prep_interrupt_lock()
1396 		desc = ioat2_get_ring_ent(ioat, ioat->head);  in ioat3_prep_interrupt_lock()
1413 	dump_desc_dbg(ioat, desc);  in ioat3_prep_interrupt_lock()