• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /* $FreeBSD: releng/12.2/sys/dev/usb/controller/ehci.c 327173 2017-12-25 04:48:39Z kan $ */
2 /*-
3  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
4  *
5  * Copyright (c) 2008 Hans Petter Selasky. All rights reserved.
6  * Copyright (c) 2004 The NetBSD Foundation, Inc. All rights reserved.
7  * Copyright (c) 2004 Lennart Augustsson. All rights reserved.
8  * Copyright (c) 2004 Charles M. Hannum. All rights reserved.
9  *
10  * Redistribution and use in source and binary forms, with or without
11  * modification, are permitted provided that the following conditions
12  * are met:
13  * 1. Redistributions of source code must retain the above copyright
14  *    notice, this list of conditions and the following disclaimer.
15  * 2. Redistributions in binary form must reproduce the above copyright
16  *    notice, this list of conditions and the following disclaimer in the
17  *    documentation and/or other materials provided with the distribution.
18  *
19  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
20  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
23  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
25  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
28  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29  * SUCH DAMAGE.
30  */
31 
32 /*
33  * USB Enhanced Host Controller Driver, a.k.a. USB 2.0 controller.
34  *
35  * The EHCI 0.96 spec can be found at
36  * http://developer.intel.com/technology/usb/download/ehci-r096.pdf
37  * The EHCI 1.0 spec can be found at
38  * http://developer.intel.com/technology/usb/download/ehci-r10.pdf
39  * and the USB 2.0 spec at
40  * http://www.usb.org/developers/docs/usb_20.zip
41  *
42  */
43 
44 #include "implementation/global_implementation.h"
45 #include "controller/ehci.h"
46 #include "controller/ehcireg.h"
47 
48 
49 #define	EHCI_BUS2SC(bus) \
50    ((ehci_softc_t *)(((uint8_t *)(bus)) - \
51     ((uint8_t *)&(((ehci_softc_t *)0)->sc_bus))))
52 
53 #undef	USB_DEBUG_VAR
54 #define	USB_DEBUG_VAR   ehcidebug
55 
56 #ifdef LOSCFG_USB_DEBUG
57 static int ehcidebug = 0;
58 static int ehciiaadbug = 0;
59 static int ehcilostintrbug = 0;
60 
61 static void ehci_dump_regs(ehci_softc_t *sc);
62 static void ehci_dump_sqh(ehci_softc_t *sc, ehci_qh_t *sqh);
63 
64 void
usb_ehci_debug_func(int level)65 usb_ehci_debug_func(int level)
66 {
67 	ehcidebug = level;
68 	PRINTK("The level of usb ehci debug is %d\n", level);
69 }
70 DEBUG_MODULE(ehci, usb_ehci_debug_func);
71 #endif
72 
73 SPIN_LOCK_INIT(g_usb_ehci_qh_spinlock);
74 
75 #define	EHCI_INTR_ENDPT 1
76 
77 extern const struct usb_bus_methods ehci_bus_methods;
78 extern const struct usb_pipe_methods ehci_device_bulk_methods;
79 extern const struct usb_pipe_methods ehci_device_ctrl_methods;
80 extern const struct usb_pipe_methods ehci_device_intr_methods;
81 extern const struct usb_pipe_methods ehci_device_isoc_fs_methods;
82 extern const struct usb_pipe_methods ehci_device_isoc_hs_methods;
83 
84 static void ehci_do_poll(struct usb_bus *);
85 static void ehci_device_done(struct usb_xfer *, usb_error_t);
86 static uint8_t ehci_check_transfer(struct usb_xfer *);
87 static void ehci_timeout(void *);
88 static void ehci_poll_timeout(void *);
89 
90 static void ehci_root_intr(ehci_softc_t *sc);
91 
92 struct ehci_std_temp {
93 	ehci_softc_t *sc;
94 	struct usb_page_cache *pc;
95 	ehci_qtd_t *td;
96 	ehci_qtd_t *td_next;
97 	uint32_t average;
98 	uint32_t qtd_status;
99 	uint32_t len;
100 	uint16_t max_frame_size;
101 	uint8_t	shortpkt;
102 	uint8_t	auto_data_toggle;
103 	uint8_t	setup_alt_next;
104 	uint8_t	last_frame;
105 };
106 
107 /* cb: usb_bus_mem_alloc_all_cb */
108 /* usb_bus_mem_flush_all_cb */
109 void
ehci_iterate_hw_softc(struct usb_bus * bus,usb_bus_mem_sub_cb_t * cb)110 ehci_iterate_hw_softc(struct usb_bus *bus, usb_bus_mem_sub_cb_t *cb)
111 {
112 	ehci_softc_t *sc = EHCI_BUS2SC(bus);
113 	uint32_t i;
114 
115 	cb(bus, &sc->sc_hw.pframes_pc, &sc->sc_hw.pframes_pg,
116 	    sizeof(uint32_t) * EHCI_FRAMELIST_COUNT, EHCI_FRAMELIST_ALIGN);
117 
118 	cb(bus, &sc->sc_hw.terminate_pc, &sc->sc_hw.terminate_pg,
119 	    sizeof(struct ehci_qh_sub), EHCI_QH_ALIGN);
120 
121 	cb(bus, &sc->sc_hw.async_start_pc, &sc->sc_hw.async_start_pg,
122 	    sizeof(ehci_qh_t), EHCI_QH_ALIGN);
123 
124 	for (i = 0; i != EHCI_VIRTUAL_FRAMELIST_COUNT; i++) {
125 		cb(bus, sc->sc_hw.intr_start_pc + i,
126 		    sc->sc_hw.intr_start_pg + i,
127 		    sizeof(ehci_qh_t), EHCI_QH_ALIGN);
128 	}
129 
130 	for (i = 0; i != EHCI_VIRTUAL_FRAMELIST_COUNT; i++) {
131 		cb(bus, sc->sc_hw.isoc_hs_start_pc + i,
132 		    sc->sc_hw.isoc_hs_start_pg + i,
133 		    sizeof(ehci_itd_t), EHCI_ITD_ALIGN);
134 	}
135 
136 	for (i = 0; i != EHCI_VIRTUAL_FRAMELIST_COUNT; i++) {
137 		cb(bus, sc->sc_hw.isoc_fs_start_pc + i,
138 		    sc->sc_hw.isoc_fs_start_pg + i,
139 		    sizeof(ehci_sitd_t), EHCI_SITD_ALIGN);
140 	}
141 }
142 
143 usb_error_t
ehci_reset(ehci_softc_t * sc)144 ehci_reset(ehci_softc_t *sc)
145 {
146 	uint32_t hcr;
147 	int i;
148 
149 	EOWRITE4(sc, EHCI_USBCMD, EHCI_CMD_HCRESET);
150 	for (i = 0; i < 100; i++) {
151 		usb_pause_mtx(NULL, hz / 128);
152 		hcr = EOREAD4(sc, EHCI_USBCMD) & EHCI_CMD_HCRESET;
153 		if (!hcr) {
154 			if (sc->sc_vendor_post_reset != NULL)
155 				sc->sc_vendor_post_reset(sc);
156 			return (USB_ERR_NORMAL_COMPLETION);
157 		}
158 	}
159 	device_printf(sc->sc_bus.bdev, "Reset timeout\n");
160 	return (USB_ERR_IOERROR);
161 }
162 
163 static usb_error_t
ehci_hcreset(ehci_softc_t * sc)164 ehci_hcreset(ehci_softc_t *sc)
165 {
166 	uint32_t hcr = 0;
167 	int i;
168 
169 	EOWRITE4(sc, EHCI_USBCMD, 0);	/* Halt controller */
170 	for (i = 0; i < 100; i++) {
171 		usb_pause_mtx(NULL, hz / 128);
172 		hcr = EOREAD4(sc, EHCI_USBSTS) & EHCI_STS_HCH;
173 		if (hcr)
174 			break;
175 	}
176 	if (!hcr)
177 		/*
178 		 * Fall through and try reset anyway even though
179 		 * Table 2-9 in the EHCI spec says this will result
180 		 * in undefined behavior.
181 		 */
182 		device_printf(sc->sc_bus.bdev, "stop timeout\n");
183 
184 	return (ehci_reset(sc));
185 }
186 
187 static usb_error_t
ehci_init_sub(struct ehci_softc * sc)188 ehci_init_sub(struct ehci_softc *sc)
189 {
190 	struct usb_page_search buf_res;
191 	uint32_t cparams;
192 	uint32_t hcr = 0;
193 	uint8_t i;
194 
195 	cparams = EREAD4(sc, EHCI_HCCPARAMS);
196 
197 	DPRINTF("cparams=0x%x\n", cparams);
198 
199 	if (EHCI_HCC_64BIT(cparams)) {
200 		DPRINTF("HCC uses 64-bit structures\n");
201 
202 		/* MUST clear segment register if 64 bit capable */
203 		EOWRITE4(sc, EHCI_CTRLDSSEGMENT, 0);
204 	}
205 
206 	usbd_get_page(&sc->sc_hw.pframes_pc, 0, &buf_res);
207 	EOWRITE4(sc, EHCI_PERIODICLISTBASE, buf_res.physaddr);
208 
209 	usbd_get_page(&sc->sc_hw.async_start_pc, 0, &buf_res);
210 	EOWRITE4(sc, EHCI_ASYNCLISTADDR, buf_res.physaddr | EHCI_LINK_QH);
211 
212 	/* enable interrupts */
213 	EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
214 
215 	/* turn on controller */
216 	EOWRITE4(sc, EHCI_USBCMD,
217 	    EHCI_CMD_ITC_1 |		/* 1 microframes interrupt delay */
218 	    (EOREAD4(sc, EHCI_USBCMD) & EHCI_CMD_FLS_M) |
219 	    EHCI_CMD_ASE |
220 	    EHCI_CMD_PSE |
221 	    EHCI_CMD_RS);
222 
223 	/* Take over port ownership */
224 	EOWRITE4(sc, EHCI_CONFIGFLAG, EHCI_CONF_CF);
225 
226 	for (i = 0; i < 100; i++) {
227 		usb_pause_mtx(NULL, hz / 128);
228 		hcr = EOREAD4(sc, EHCI_USBSTS) & EHCI_STS_HCH;
229 		if (!hcr) {
230 			break;
231 		}
232 	}
233 	if (hcr) {
234 		device_printf(sc->sc_bus.bdev, "run timeout\n");
235 		return (USB_ERR_IOERROR);
236 	}
237 	return (USB_ERR_NORMAL_COMPLETION);
238 }
239 
240 usb_error_t
ehci_init(ehci_softc_t * sc)241 ehci_init(ehci_softc_t *sc)
242 {
243 	struct usb_page_search buf_res;
244 	struct ehci_qh_sub *qh_sub;
245 	ehci_qh_t *qh_t;
246 	uint32_t version;
247 	uint32_t sparams;
248 	uint32_t *pframes;
249 	uint16_t i;
250 	uint16_t x;
251 	uint16_t y;
252 	uint16_t bit;
253 	usb_error_t err = USB_ERR_NORMAL_COMPLETION;
254 
255 	DPRINTF("start\n");
256 
257 	callout_init_mtx(&sc->sc_tmo_pcd, &sc->sc_bus.bus_mtx, 0);
258 	callout_init_mtx(&sc->sc_tmo_poll, &sc->sc_bus.bus_mtx, 0);
259 
260 	sc->sc_offs = EHCI_CAPLENGTH(EREAD4(sc, EHCI_CAPLEN_HCIVERSION));
261 
262 #ifdef LOSCFG_USB_DEBUG
263 	if (ehciiaadbug)
264 		sc->sc_flags |= EHCI_SCFLG_IAADBUG;
265 	if (ehcilostintrbug)
266 		sc->sc_flags |= EHCI_SCFLG_LOSTINTRBUG;
267 	if (ehcidebug > 2) {
268 		ehci_dump_regs(sc);
269 	}
270 #endif
271 
272 	version = EHCI_HCIVERSION(EREAD4(sc, EHCI_CAPLEN_HCIVERSION));
273 	device_printf(sc->sc_bus.bdev, "EHCI version %x.%x\n",
274 	    version >> 8, version & 0xff);
275 
276 	sparams = EREAD4(sc, EHCI_HCSPARAMS);
277 	DPRINTF("sparams=0x%x\n", sparams);
278 
279 	sc->sc_noport = EHCI_HCS_N_PORTS(sparams);
280 	sc->sc_bus.usbrev = USB_REV_2_0;
281 	if (!(sc->sc_flags & EHCI_SCFLG_DONTRESET)) {
282 		/* Reset the controller */
283 		DPRINTF("%s: resetting\n",
284 		    device_get_nameunit(sc->sc_bus.bdev));
285 		err = ehci_hcreset(sc);
286 		if (err) {
287 			device_printf(sc->sc_bus.bdev, "reset timeout\n");
288 			return (err);
289 		}
290 	}
291 
292 	/*
293 	 * use current frame-list-size selection 0: 1024*4 bytes 1:  512*4
294 	 * bytes 2:  256*4 bytes 3:	  unknown
295 	 */
296 	if (EHCI_CMD_FLS(EOREAD4(sc, EHCI_USBCMD)) == 3) {
297 		device_printf(sc->sc_bus.bdev, "invalid frame-list-size\n");
298 		return (USB_ERR_IOERROR);
299 	}
300 	/* set up the bus struct */
301 	sc->sc_bus.methods = &ehci_bus_methods;
302 
303 	sc->sc_eintrs = EHCI_NORMAL_INTRS;
304 
305 	usbd_get_page(&sc->sc_hw.terminate_pc, 0, &buf_res);
306 	qh_sub = (struct ehci_qh_sub *)buf_res.buffer;
307 
308 	sc->sc_terminate_self = htohc32(sc, buf_res.physaddr);
309 
310 	/* init terminate TD */
311 	qh_sub->qtd_next =
312 	    htohc32(sc, EHCI_LINK_TERMINATE);
313 	qh_sub->qtd_altnext =
314 	    htohc32(sc, EHCI_LINK_TERMINATE);
315 	qh_sub->qtd_status =
316 	    htohc32(sc, EHCI_QTD_HALTED);
317 
318 	for (i = 0; i < EHCI_VIRTUAL_FRAMELIST_COUNT; i++) {
319 		usbd_get_page(sc->sc_hw.intr_start_pc + i, 0, &buf_res);
320 
321 		qh_t = (ehci_qh_t *)buf_res.buffer;
322 
323 		/* initialize page cache pointer */
324 
325 		qh_t->page_cache = sc->sc_hw.intr_start_pc + i;
326 
327 		/* store a pointer to queue head */
328 
329 		sc->sc_intr_p_last[i] = qh_t;
330 
331 		qh_t->qh_self =
332 		    htohc32(sc, buf_res.physaddr) |
333 		    htohc32(sc, EHCI_LINK_QH);
334 
335 		qh_t->qh_endp =
336 		    htohc32(sc, EHCI_QH_SET_EPS(EHCI_QH_SPEED_HIGH));
337 		qh_t->qh_endphub =
338 		    htohc32(sc, EHCI_QH_SET_MULT(1));
339 		qh_t->qh_curqtd = 0;
340 
341 		qh_t->qh_qtd.qtd_next =
342 		    htohc32(sc, EHCI_LINK_TERMINATE);
343 		qh_t->qh_qtd.qtd_altnext =
344 		    htohc32(sc, EHCI_LINK_TERMINATE);
345 		qh_t->qh_qtd.qtd_status =
346 		    htohc32(sc, EHCI_QTD_HALTED);
347 	}
348 
349 	/*
350 	 * the QHs are arranged to give poll intervals that are
351 	 * powers of 2 times 1ms
352 	 */
353 	bit = EHCI_VIRTUAL_FRAMELIST_COUNT / 2;
354 	while (bit) {
355 		x = bit;
356 		while (x & bit) {
357 			ehci_qh_t *qh_x;
358 			ehci_qh_t *qh_y;
359 
360 			y = (x ^ bit) | (bit / 2);
361 
362 			qh_x = sc->sc_intr_p_last[x];
363 			qh_y = sc->sc_intr_p_last[y];
364 
365 			/*
366 			 * the next QH has half the poll interval
367 			 */
368 			qh_x->qh_link = qh_y->qh_self;
369 
370 			x++;
371 		}
372 		bit >>= 1;
373 	}
374 
375 	qh_t = sc->sc_intr_p_last[0];
376 
377 	/* the last (1ms) QH terminates */
378 	qh_t->qh_link = htohc32(sc, EHCI_LINK_TERMINATE);
379 
380 	for (i = 0; i < EHCI_VIRTUAL_FRAMELIST_COUNT; i++) {
381 		ehci_sitd_t *sitd;
382 		ehci_itd_t *itd;
383 
384 		usbd_get_page(sc->sc_hw.isoc_fs_start_pc + i, 0, &buf_res);
385 		sitd = (ehci_sitd_t *)buf_res.buffer;
386 
387 		/* initialize page cache pointer */
388 
389 		sitd->page_cache = sc->sc_hw.isoc_fs_start_pc + i;
390 
391 		/* store a pointer to the transfer descriptor */
392 
393 		sc->sc_isoc_fs_p_last[i] = sitd;
394 
395 		/* initialize full speed isochronous */
396 
397 		sitd->sitd_self =
398 		    htohc32(sc, buf_res.physaddr) |
399 		    htohc32(sc, EHCI_LINK_SITD);
400 
401 		sitd->sitd_back =
402 		    htohc32(sc, EHCI_LINK_TERMINATE);
403 
404 		sitd->sitd_next =
405 		    sc->sc_intr_p_last[i | (EHCI_VIRTUAL_FRAMELIST_COUNT / 2)]->qh_self;
406 
407 
408 		usbd_get_page(sc->sc_hw.isoc_hs_start_pc + i, 0, &buf_res);
409 
410 		itd = (ehci_itd_t *)buf_res.buffer;
411 
412 		/* initialize page cache pointer */
413 
414 		itd->page_cache = sc->sc_hw.isoc_hs_start_pc + i;
415 
416 		/* store a pointer to the transfer descriptor */
417 
418 		sc->sc_isoc_hs_p_last[i] = itd;
419 
420 		/* initialize high speed isochronous */
421 
422 		itd->itd_self =
423 		    htohc32(sc, buf_res.physaddr) |
424 		    htohc32(sc, EHCI_LINK_ITD);
425 
426 		itd->itd_next =
427 		    sitd->sitd_self;
428 	}
429 
430 	usbd_get_page(&sc->sc_hw.pframes_pc, 0, &buf_res);
431 
432 	pframes = (uint32_t *)buf_res.buffer;
433 
434 	/*
435 	 * execution order:
436 	 * pframes -> high speed isochronous ->
437 	 *    full speed isochronous -> interrupt QH's
438 	 */
439 	for (i = 0; i < EHCI_FRAMELIST_COUNT; i++) {
440 		pframes[i] = sc->sc_isoc_hs_p_last
441 		    [i & (EHCI_VIRTUAL_FRAMELIST_COUNT - 1)]->itd_self;
442 	}
443 
444 	usbd_get_page(&sc->sc_hw.async_start_pc, 0, &buf_res);
445 
446 	qh_t = (ehci_qh_t *)buf_res.buffer;
447 
448 	/* initialize page cache pointer */
449 
450 	qh_t->page_cache = &sc->sc_hw.async_start_pc;
451 
452 	/* store a pointer to the queue head */
453 
454 	sc->sc_async_p_last = qh_t;
455 
456 	/* init dummy QH that starts the async list */
457 
458 	qh_t->qh_self =
459 	    htohc32(sc, buf_res.physaddr) |
460 	    htohc32(sc, EHCI_LINK_QH);
461 
462 	/* fill the QH */
463 	qh_t->qh_endp =
464 	    htohc32(sc, EHCI_QH_SET_EPS(EHCI_QH_SPEED_HIGH) | EHCI_QH_HRECL);
465 	qh_t->qh_endphub = htohc32(sc, EHCI_QH_SET_MULT(1));
466 	qh_t->qh_link = qh_t->qh_self;
467 	qh_t->qh_curqtd = 0;
468 
469 	/* fill the overlay qTD */
470 	qh_t->qh_qtd.qtd_next = htohc32(sc, EHCI_LINK_TERMINATE);
471 	qh_t->qh_qtd.qtd_altnext = htohc32(sc, EHCI_LINK_TERMINATE);
472 	qh_t->qh_qtd.qtd_status = htohc32(sc, EHCI_QTD_HALTED);
473 
474 	/* flush all cache into memory */
475 
476 	usb_bus_mem_flush_all(&sc->sc_bus, &ehci_iterate_hw_softc);
477 
478 #ifdef LOSCFG_USB_DEBUG
479 	if (ehcidebug) {
480 		ehci_dump_sqh(sc, sc->sc_async_p_last);
481 	}
482 #endif
483 
484 	/* finial setup */
485 	err = ehci_init_sub(sc);
486 
487 	if (!err) {
488 		/* catch any lost interrupts */
489 		ehci_do_poll(&sc->sc_bus);
490 	}
491 	return (err);
492 }
493 
494 /*
495  * shut down the controller when the system is going down
496  */
497 void
ehci_detach(ehci_softc_t * sc)498 ehci_detach(ehci_softc_t *sc)
499 {
500 	USB_BUS_LOCK(&sc->sc_bus);
501 
502 	callout_stop(&sc->sc_tmo_pcd);
503 	callout_stop(&sc->sc_tmo_poll);
504 
505 	EOWRITE4(sc, EHCI_USBINTR, 0);
506 	USB_BUS_UNLOCK(&sc->sc_bus);
507 
508 	if (ehci_hcreset(sc)) {
509 		DPRINTF("reset failed!\n");
510 	}
511 
512 	/* XXX let stray task complete */
513 	usb_pause_mtx(NULL, hz / 20);
514 
515 	callout_drain(&sc->sc_tmo_pcd);
516 	callout_drain(&sc->sc_tmo_poll);
517 }
518 
519 static void
ehci_suspend(ehci_softc_t * sc)520 ehci_suspend(ehci_softc_t *sc)
521 {
522 	DPRINTF("stopping the HC\n");
523 
524 	/* reset HC */
525 	(void)ehci_hcreset(sc);
526 }
527 
528 static void
ehci_resume(ehci_softc_t * sc)529 ehci_resume(ehci_softc_t *sc)
530 {
531 	/* reset HC */
532 	(void)ehci_hcreset(sc);
533 
534 	/* setup HC */
535 	(void)ehci_init_sub(sc);
536 
537 	/* catch any lost interrupts */
538 	ehci_do_poll(&sc->sc_bus);
539 }
540 
541 #ifdef LOSCFG_USB_DEBUG
542 static void
ehci_dump_regs(ehci_softc_t * sc)543 ehci_dump_regs(ehci_softc_t *sc)
544 {
545 	uint32_t i;
546 
547 	i = EOREAD4(sc, EHCI_USBCMD);
548 	PRINTK("cmd=0x%08x\n", i);
549 
550 	if (i & EHCI_CMD_ITC_1)
551 		PRINTK(" EHCI_CMD_ITC_1\n");
552 	if (i & EHCI_CMD_ITC_2)
553 		PRINTK(" EHCI_CMD_ITC_2\n");
554 	if (i & EHCI_CMD_ITC_4)
555 		PRINTK(" EHCI_CMD_ITC_4\n");
556 	if (i & EHCI_CMD_ITC_8)
557 		PRINTK(" EHCI_CMD_ITC_8\n");
558 	if (i & EHCI_CMD_ITC_16)
559 		PRINTK(" EHCI_CMD_ITC_16\n");
560 	if (i & EHCI_CMD_ITC_32)
561 		PRINTK(" EHCI_CMD_ITC_32\n");
562 	if (i & EHCI_CMD_ITC_64)
563 		PRINTK(" EHCI_CMD_ITC_64\n");
564 	if (i & EHCI_CMD_ASPME)
565 		PRINTK(" EHCI_CMD_ASPME\n");
566 	if (i & EHCI_CMD_ASPMC)
567 		PRINTK(" EHCI_CMD_ASPMC\n");
568 	if (i & EHCI_CMD_LHCR)
569 		PRINTK(" EHCI_CMD_LHCR\n");
570 	if (i & EHCI_CMD_IAAD)
571 		PRINTK(" EHCI_CMD_IAAD\n");
572 	if (i & EHCI_CMD_ASE)
573 		PRINTK(" EHCI_CMD_ASE\n");
574 	if (i & EHCI_CMD_PSE)
575 		PRINTK(" EHCI_CMD_PSE\n");
576 	if (i & EHCI_CMD_FLS_M)
577 		PRINTK(" EHCI_CMD_FLS_M\n");
578 	if (i & EHCI_CMD_HCRESET)
579 		PRINTK(" EHCI_CMD_HCRESET\n");
580 	if (i & EHCI_CMD_RS)
581 		PRINTK(" EHCI_CMD_RS\n");
582 
583 	i = EOREAD4(sc, EHCI_USBSTS);
584 
585 	PRINTK("sts=0x%08x\n", i);
586 
587 	if (i & EHCI_STS_ASS)
588 		PRINTK(" EHCI_STS_ASS\n");
589 	if (i & EHCI_STS_PSS)
590 		PRINTK(" EHCI_STS_PSS\n");
591 	if (i & EHCI_STS_REC)
592 		PRINTK(" EHCI_STS_REC\n");
593 	if (i & EHCI_STS_HCH)
594 		PRINTK(" EHCI_STS_HCH\n");
595 	if (i & EHCI_STS_IAA)
596 		PRINTK(" EHCI_STS_IAA\n");
597 	if (i & EHCI_STS_HSE)
598 		PRINTK(" EHCI_STS_HSE\n");
599 	if (i & EHCI_STS_FLR)
600 		PRINTK(" EHCI_STS_FLR\n");
601 	if (i & EHCI_STS_PCD)
602 		PRINTK(" EHCI_STS_PCD\n");
603 	if (i & EHCI_STS_ERRINT)
604 		PRINTK(" EHCI_STS_ERRINT\n");
605 	if (i & EHCI_STS_INT)
606 		PRINTK(" EHCI_STS_INT\n");
607 
608 	PRINTK("intr=0x%08x\n",
609 	    EOREAD4(sc, EHCI_USBINTR));
610 	PRINTK("frindex=0x%08x ctrdsegm=0x%08x periodic=0x%08x async=0x%08x\n",
611 	    EOREAD4(sc, EHCI_FRINDEX),
612 	    EOREAD4(sc, EHCI_CTRLDSSEGMENT),
613 	    EOREAD4(sc, EHCI_PERIODICLISTBASE),
614 	    EOREAD4(sc, EHCI_ASYNCLISTADDR));
615 	for (i = 1; i <= sc->sc_noport; i++) {
616 		PRINTK("port %d status=0x%08x\n", i,
617 		    EOREAD4(sc, EHCI_PORTSC(i)));
618 	}
619 }
620 
621 static void
ehci_dump_link(ehci_softc_t * sc,uint32_t link,int type)622 ehci_dump_link(ehci_softc_t *sc, uint32_t link, int type)
623 {
624 	link = hc32toh(sc, link);
625 	PRINTK("0x%08x", link);
626 	if (link & EHCI_LINK_TERMINATE)
627 		PRINTK("<T>");
628 	else {
629 		PRINTK("<");
630 		if (type) {
631 			switch (EHCI_LINK_TYPE(link)) {
632 			case EHCI_LINK_ITD:
633 				PRINTK("ITD");
634 				break;
635 			case EHCI_LINK_QH:
636 				PRINTK("QH");
637 				break;
638 			case EHCI_LINK_SITD:
639 				PRINTK("SITD");
640 				break;
641 			case EHCI_LINK_FSTN:
642 				PRINTK("FSTN");
643 				break;
644 			}
645 		}
646 		PRINTK(">");
647 	}
648 }
649 
650 static void
ehci_dump_qtd(ehci_softc_t * sc,ehci_qtd_t * qtd)651 ehci_dump_qtd(ehci_softc_t *sc, ehci_qtd_t *qtd)
652 {
653 	uint32_t s;
654 
655 	PRINTK("  next=");
656 	ehci_dump_link(sc, qtd->qtd_next, 0);
657 	PRINTK(" altnext=");
658 	ehci_dump_link(sc, qtd->qtd_altnext, 0);
659 	PRINTK("\n");
660 	s = hc32toh(sc, qtd->qtd_status);
661 	PRINTK("  status=0x%08x: toggle=%d bytes=0x%x ioc=%d c_page=0x%x\n",
662 	    s, EHCI_QTD_GET_TOGGLE(s), EHCI_QTD_GET_BYTES(s),
663 	    EHCI_QTD_GET_IOC(s), EHCI_QTD_GET_C_PAGE(s));
664 	PRINTK("    cerr=%d pid=%d stat=%s%s%s%s%s%s%s%s\n",
665 	    EHCI_QTD_GET_CERR(s), EHCI_QTD_GET_PID(s),
666 	    (s & EHCI_QTD_ACTIVE) ? "ACTIVE" : "NOT_ACTIVE",
667 	    (s & EHCI_QTD_HALTED) ? "-HALTED" : "",
668 	    (s & EHCI_QTD_BUFERR) ? "-BUFERR" : "",
669 	    (s & EHCI_QTD_BABBLE) ? "-BABBLE" : "",
670 	    (s & EHCI_QTD_XACTERR) ? "-XACTERR" : "",
671 	    (s & EHCI_QTD_MISSEDMICRO) ? "-MISSED" : "",
672 	    (s & EHCI_QTD_SPLITXSTATE) ? "-SPLIT" : "",
673 	    (s & EHCI_QTD_PINGSTATE) ? "-PING" : "");
674 
675 	for (s = 0; s < 5; s++) {
676 		PRINTK("  buffer[%d]=0x%08x\n", s,
677 		    hc32toh(sc, qtd->qtd_buffer[s]));
678 	}
679 	for (s = 0; s < 5; s++) {
680 		PRINTK("  buffer_hi[%d]=0x%08x\n", s,
681 		    hc32toh(sc, qtd->qtd_buffer_hi[s]));
682 	}
683 }
684 
685 static uint8_t
ehci_dump_sqtd(ehci_softc_t * sc,ehci_qtd_t * sqtd)686 ehci_dump_sqtd(ehci_softc_t *sc, ehci_qtd_t *sqtd)
687 {
688 	uint8_t temp;
689 
690 	usb_pc_cpu_invalidate(sqtd->page_cache);
691 	PRINTK("QTD(%p) at 0x%08x:\n", sqtd, hc32toh(sc, sqtd->qtd_self));
692 	ehci_dump_qtd(sc, sqtd);
693 	temp = (sqtd->qtd_next & htohc32(sc, EHCI_LINK_TERMINATE)) ? 1 : 0;
694 	return (temp);
695 }
696 
697 static void
ehci_dump_sqtds(ehci_softc_t * sc,ehci_qtd_t * sqtd)698 ehci_dump_sqtds(ehci_softc_t *sc, ehci_qtd_t *sqtd)
699 {
700 	uint16_t i;
701 	uint8_t stop;
702 
703 	stop = 0;
704 	for (i = 0; sqtd && (i < 20) && !stop; sqtd = sqtd->obj_next, i++) {
705 		stop = ehci_dump_sqtd(sc, sqtd);
706 	}
707 	if (sqtd) {
708 		PRINTK("dump aborted, too many TDs\n");
709 	}
710 }
711 
712 static void
ehci_dump_sqh(ehci_softc_t * sc,ehci_qh_t * qh)713 ehci_dump_sqh(ehci_softc_t *sc, ehci_qh_t *qh)
714 {
715 	uint32_t endp;
716 	uint32_t endphub;
717 
718 	usb_pc_cpu_invalidate(qh->page_cache);
719 	PRINTK("QH(%p) at 0x%08x:\n", qh, hc32toh(sc, qh->qh_self) & ~0x1F);
720 	PRINTK("  link=");
721 	ehci_dump_link(sc, qh->qh_link, 1);
722 	PRINTK("\n");
723 	endp = hc32toh(sc, qh->qh_endp);
724 	PRINTK("  endp=0x%08x\n", endp);
725 	PRINTK("    addr=0x%02x inact=%d endpt=%d eps=%d dtc=%d hrecl=%d\n",
726 	    EHCI_QH_GET_ADDR(endp), EHCI_QH_GET_INACT(endp),
727 	    EHCI_QH_GET_ENDPT(endp), EHCI_QH_GET_EPS(endp),
728 	    EHCI_QH_GET_DTC(endp), EHCI_QH_GET_HRECL(endp));
729 	PRINTK("    mpl=0x%x ctl=%d nrl=%d\n",
730 	    EHCI_QH_GET_MPL(endp), EHCI_QH_GET_CTL(endp),
731 	    EHCI_QH_GET_NRL(endp));
732 	endphub = hc32toh(sc, qh->qh_endphub);
733 	PRINTK("  endphub=0x%08x\n", endphub);
734 	PRINTK("    smask=0x%02x cmask=0x%02x huba=0x%02x port=%d mult=%d\n",
735 	    EHCI_QH_GET_SMASK(endphub), EHCI_QH_GET_CMASK(endphub),
736 	    EHCI_QH_GET_HUBA(endphub), EHCI_QH_GET_PORT(endphub),
737 	    EHCI_QH_GET_MULT(endphub));
738 	PRINTK("  curqtd=");
739 	ehci_dump_link(sc, qh->qh_curqtd, 0);
740 	PRINTK("\n");
741 	PRINTK("Overlay qTD:\n");
742 	ehci_dump_qtd(sc, (void *)&qh->qh_qtd);
743 }
744 
745 static void
ehci_dump_sitd(ehci_softc_t * sc,ehci_sitd_t * sitd)746 ehci_dump_sitd(ehci_softc_t *sc, ehci_sitd_t *sitd)
747 {
748 	usb_pc_cpu_invalidate(sitd->page_cache);
749 	PRINTK("SITD(%p) at 0x%08x\n", sitd, hc32toh(sc, sitd->sitd_self) & ~0x1F);
750 	PRINTK(" next=0x%08x\n", hc32toh(sc, sitd->sitd_next));
751 	PRINTK(" portaddr=0x%08x dir=%s addr=%d endpt=0x%x port=0x%x huba=0x%x\n",
752 	    hc32toh(sc, sitd->sitd_portaddr),
753 	    (sitd->sitd_portaddr & htohc32(sc, EHCI_SITD_SET_DIR_IN))
754 	    ? "in" : "out",
755 	    EHCI_SITD_GET_ADDR(hc32toh(sc, sitd->sitd_portaddr)),
756 	    EHCI_SITD_GET_ENDPT(hc32toh(sc, sitd->sitd_portaddr)),
757 	    EHCI_SITD_GET_PORT(hc32toh(sc, sitd->sitd_portaddr)),
758 	    EHCI_SITD_GET_HUBA(hc32toh(sc, sitd->sitd_portaddr)));
759 	PRINTK(" mask=0x%08x\n", hc32toh(sc, sitd->sitd_mask));
760 	PRINTK(" status=0x%08x <%s> len=0x%x\n", hc32toh(sc, sitd->sitd_status),
761 	    (sitd->sitd_status & htohc32(sc, EHCI_SITD_ACTIVE)) ? "ACTIVE" : "",
762 	    EHCI_SITD_GET_LEN(hc32toh(sc, sitd->sitd_status)));
763 	PRINTK(" back=0x%08x, bp=0x%08x,0x%08x,0x%08x,0x%08x\n",
764 	    hc32toh(sc, sitd->sitd_back),
765 	    hc32toh(sc, sitd->sitd_bp[0]),
766 	    hc32toh(sc, sitd->sitd_bp[1]),
767 	    hc32toh(sc, sitd->sitd_bp_hi[0]),
768 	    hc32toh(sc, sitd->sitd_bp_hi[1]));
769 }
770 
771 static void
ehci_dump_itd(ehci_softc_t * sc,ehci_itd_t * itd)772 ehci_dump_itd(ehci_softc_t *sc, ehci_itd_t *itd)
773 {
774 	usb_pc_cpu_invalidate(itd->page_cache);
775 	PRINTK("ITD(%p) at 0x%08x\n", itd, hc32toh(sc, itd->itd_self) & ~0x1F);
776 	PRINTK(" next=0x%08x\n", hc32toh(sc, itd->itd_next));
777 	PRINTK(" status[0]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[0]),
778 	    (itd->itd_status[0] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
779 	PRINTK(" status[1]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[1]),
780 	    (itd->itd_status[1] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
781 	PRINTK(" status[2]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[2]),
782 	    (itd->itd_status[2] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
783 	PRINTK(" status[3]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[3]),
784 	    (itd->itd_status[3] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
785 	PRINTK(" status[4]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[4]),
786 	    (itd->itd_status[4] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
787 	PRINTK(" status[5]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[5]),
788 	    (itd->itd_status[5] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
789 	PRINTK(" status[6]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[6]),
790 	    (itd->itd_status[6] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
791 	PRINTK(" status[7]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[7]),
792 	    (itd->itd_status[7] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
793 	PRINTK(" bp[0]=0x%08x\n", hc32toh(sc, itd->itd_bp[0]));
794 	PRINTK("  addr=0x%02x; endpt=0x%01x\n",
795 	    EHCI_ITD_GET_ADDR(hc32toh(sc, itd->itd_bp[0])),
796 	    EHCI_ITD_GET_ENDPT(hc32toh(sc, itd->itd_bp[0])));
797 	PRINTK(" bp[1]=0x%08x\n", hc32toh(sc, itd->itd_bp[1]));
798 	PRINTK(" dir=%s; mpl=0x%02x\n",
799 	    (hc32toh(sc, itd->itd_bp[1]) & EHCI_ITD_SET_DIR_IN) ? "in" : "out",
800 	    EHCI_ITD_GET_MPL(hc32toh(sc, itd->itd_bp[1])));
801 	PRINTK(" bp[2..6]=0x%08x,0x%08x,0x%08x,0x%08x,0x%08x\n",
802 	    hc32toh(sc, itd->itd_bp[2]),
803 	    hc32toh(sc, itd->itd_bp[3]),
804 	    hc32toh(sc, itd->itd_bp[4]),
805 	    hc32toh(sc, itd->itd_bp[5]),
806 	    hc32toh(sc, itd->itd_bp[6]));
807 	PRINTK(" bp_hi=0x%08x,0x%08x,0x%08x,0x%08x,\n"
808 	    "      0x%08x,0x%08x,0x%08x\n",
809 	    hc32toh(sc, itd->itd_bp_hi[0]),
810 	    hc32toh(sc, itd->itd_bp_hi[1]),
811 	    hc32toh(sc, itd->itd_bp_hi[2]),
812 	    hc32toh(sc, itd->itd_bp_hi[3]),
813 	    hc32toh(sc, itd->itd_bp_hi[4]),
814 	    hc32toh(sc, itd->itd_bp_hi[5]),
815 	    hc32toh(sc, itd->itd_bp_hi[6]));
816 }
817 
818 static void
ehci_dump_isoc(ehci_softc_t * sc)819 ehci_dump_isoc(ehci_softc_t *sc)
820 {
821 	ehci_itd_t *itd;
822 	ehci_sitd_t *sitd;
823 	uint16_t max = 1000;
824 	uint16_t pos;
825 
826 	pos = (EOREAD4(sc, EHCI_FRINDEX) / 8) &
827 	    (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
828 
829 	PRINTK("%s: isochronous dump from frame 0x%03x:\n",
830 	    __FUNCTION__, pos);
831 
832 	itd = sc->sc_isoc_hs_p_last[pos];
833 	sitd = sc->sc_isoc_fs_p_last[pos];
834 
835 	while (itd && max && max--) {
836 		ehci_dump_itd(sc, itd);
837 		itd = itd->prev;
838 	}
839 
840 	while (sitd && max && max--) {
841 		ehci_dump_sitd(sc, sitd);
842 		sitd = sitd->prev;
843 	}
844 }
845 
846 #endif
847 
848 static void
ehci_transfer_intr_enqueue(struct usb_xfer * xfer)849 ehci_transfer_intr_enqueue(struct usb_xfer *xfer)
850 {
851 	/* check for early completion */
852 	if (ehci_check_transfer(xfer)) {
853 		DPRINTFN(0, " ehci_check_transfer return\n");
854 		return;
855 	}
856 	DPRINTFN(7, " enqueue\n");
857 	/* put transfer on interrupt queue */
858 	usbd_transfer_enqueue(&xfer->xroot->bus->intr_q, xfer);
859 
860 	/* start timeout, if any */
861 	if (xfer->timeout != 0) {
862 		usbd_transfer_timeout_ms(xfer, &ehci_timeout, xfer->timeout);
863 	}
864 }
865 
866 #define	EHCI_APPEND_FS_TD(std, last) (last) = _ehci_append_fs_td(std, last)
867 static ehci_sitd_t *
_ehci_append_fs_td(ehci_sitd_t * ehci_std,ehci_sitd_t * last)868 _ehci_append_fs_td(ehci_sitd_t *ehci_std, ehci_sitd_t *last)
869 {
870 	DPRINTFN(11, "%p to %p\n", ehci_std, last);
871 
872 	/* (sc->sc_bus.mtx) must be locked */
873 
874 	ehci_std->next = last->next;
875 	ehci_std->sitd_next = last->sitd_next;
876 
877 	ehci_std->prev = last;
878 
879 	usb_pc_cpu_flush(ehci_std->page_cache);
880 
881 	/*
882 	 * the last->next->prev is never followed: std->next->prev = std;
883 	 */
884 	last->next = ehci_std;
885 	last->sitd_next = ehci_std->sitd_self;
886 
887 	usb_pc_cpu_flush(last->page_cache);
888 
889 	return (ehci_std);
890 }
891 
892 #define	EHCI_APPEND_HS_TD(std, last) (last) = _ehci_append_hs_td(std, last)
893 static ehci_itd_t *
_ehci_append_hs_td(ehci_itd_t * ehci_std,ehci_itd_t * last)894 _ehci_append_hs_td(ehci_itd_t *ehci_std, ehci_itd_t *last)
895 {
896 	DPRINTFN(11, "%p to %p\n", ehci_std, last);
897 
898 	/* (sc->sc_bus.mtx) must be locked */
899 
900 	ehci_std->next = last->next;
901 	ehci_std->itd_next = last->itd_next;
902 
903 	ehci_std->prev = last;
904 
905 	usb_pc_cpu_flush(ehci_std->page_cache);
906 
907 	/*
908 	 * the last->next->prev is never followed: std->next->prev = std;
909 	 */
910 	last->next = ehci_std;
911 	last->itd_next = ehci_std->itd_self;
912 
913 	usb_pc_cpu_flush(last->page_cache);
914 
915 	return (ehci_std);
916 }
917 
918 #define	EHCI_APPEND_QH(sqh, last) (last) = _ehci_append_qh(sqh, last)
919 static ehci_qh_t *
_ehci_append_qh(ehci_qh_t * sqh,ehci_qh_t * last)920 _ehci_append_qh(ehci_qh_t *sqh, ehci_qh_t *last)
921 {
922 	DPRINTFN(11, "%p to %p\n", sqh, last);
923 
924 	if (sqh->prev != NULL) {
925 		/* should not happen */
926 		DPRINTFN(0, "QH already linked!\n");
927 		return (last);
928 	}
929 	/* (sc->sc_bus.mtx) must be locked */
930 
931 	sqh->next = last->next;
932 	sqh->qh_link = last->qh_link;
933 
934 	sqh->prev = last;
935 
936 	usb_pc_cpu_flush(sqh->page_cache);
937 
938 	/*
939 	 * the last->next->prev is never followed: sqh->next->prev = sqh;
940 	 */
941 
942 	last->next = sqh;
943 	last->qh_link = sqh->qh_self;
944 
945 	usb_pc_cpu_flush(last->page_cache);
946 
947 	return (sqh);
948 }
949 
950 #define	EHCI_REMOVE_FS_TD(std, last) (last) = _ehci_remove_fs_td(std, last)
951 static ehci_sitd_t *
_ehci_remove_fs_td(ehci_sitd_t * ehci_std,ehci_sitd_t * last)952 _ehci_remove_fs_td(ehci_sitd_t *ehci_std, ehci_sitd_t *last)
953 {
954 	DPRINTFN(11, "%p from %p\n", ehci_std, last);
955 
956 	/* (sc->sc_bus.mtx) must be locked */
957 
958 	ehci_std->prev->next = ehci_std->next;
959 	ehci_std->prev->sitd_next = ehci_std->sitd_next;
960 
961 	usb_pc_cpu_flush(ehci_std->prev->page_cache);
962 
963 	if (ehci_std->next) {
964 		ehci_std->next->prev = ehci_std->prev;
965 		usb_pc_cpu_flush(ehci_std->next->page_cache);
966 	}
967 	return ((last == ehci_std) ? ehci_std->prev : last);
968 }
969 
970 #define	EHCI_REMOVE_HS_TD(std, last) (last) = _ehci_remove_hs_td(std, last)
971 static ehci_itd_t *
_ehci_remove_hs_td(ehci_itd_t * ehci_std,ehci_itd_t * last)972 _ehci_remove_hs_td(ehci_itd_t *ehci_std, ehci_itd_t *last)
973 {
974 	DPRINTFN(11, "%p from %p\n", ehci_std, last);
975 
976 	/* (sc->sc_bus.mtx) must be locked */
977 
978 	ehci_std->prev->next = ehci_std->next;
979 	ehci_std->prev->itd_next = ehci_std->itd_next;
980 
981 	usb_pc_cpu_flush(ehci_std->prev->page_cache);
982 
983 	if (ehci_std->next) {
984 		ehci_std->next->prev = ehci_std->prev;
985 		usb_pc_cpu_flush(ehci_std->next->page_cache);
986 	}
987 	return ((last == ehci_std) ? ehci_std->prev : last);
988 }
989 
990 #define	EHCI_REMOVE_QH(sqh, last) (last) = _ehci_remove_qh(sqh, last)
991 static ehci_qh_t *
_ehci_remove_qh(ehci_qh_t * sqh,ehci_qh_t * last)992 _ehci_remove_qh(ehci_qh_t *sqh, ehci_qh_t *last)
993 {
994 	DPRINTFN(11, "%p from %p\n", sqh, last);
995 
996 	/* (sc->sc_bus.mtx) must be locked */
997 
998 	/* only remove if not removed from a queue */
999 	if (sqh->prev) {
1000 
1001 		sqh->prev->next = sqh->next;
1002 		sqh->prev->qh_link = sqh->qh_link;
1003 
1004 		usb_pc_cpu_flush(sqh->prev->page_cache);
1005 
1006 		if (sqh->next) {
1007 			sqh->next->prev = sqh->prev;
1008 			usb_pc_cpu_flush(sqh->next->page_cache);
1009 		}
1010 		last = ((last == sqh) ? sqh->prev : last);
1011 
1012 		sqh->prev = 0;
1013 
1014 		usb_pc_cpu_flush(sqh->page_cache);
1015 	}
1016 	return (last);
1017 }
1018 
1019 static void
ehci_data_toggle_update(struct usb_xfer * xfer,uint16_t actlen,uint16_t xlen)1020 ehci_data_toggle_update(struct usb_xfer *xfer, uint16_t actlen, uint16_t xlen)
1021 {
1022 	uint16_t rem;
1023 	uint8_t dt;
1024 
1025 	/* count number of full packets */
1026 	dt = (actlen / xfer->max_packet_size) & 1;
1027 
1028 	/* compute remainder */
1029 	rem = actlen % xfer->max_packet_size;
1030 
1031 	if (rem > 0)
1032 		dt ^= 1;	/* short packet at the end */
1033 	else if (actlen != xlen)
1034 		dt ^= 1;	/* zero length packet at the end */
1035 	else if (xlen == 0)
1036 		dt ^= 1;	/* zero length transfer */
1037 
1038 	xfer->endpoint->toggle_next ^= dt;
1039 }
1040 
1041 static usb_error_t
ehci_non_isoc_done_sub(struct usb_xfer * xfer)1042 ehci_non_isoc_done_sub(struct usb_xfer *xfer)
1043 {
1044 	ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
1045 	ehci_qtd_t *td;
1046 	ehci_qtd_t *td_alt_next;
1047 	uint32_t status;
1048 	uint16_t len;
1049 
1050 	td = (ehci_qtd_t *)xfer->td_transfer_cache;
1051 	td_alt_next = td->alt_next;
1052 
1053 	if (xfer->aframes != xfer->nframes) {
1054 		usbd_xfer_set_frame_len(xfer, xfer->aframes, 0);
1055 	}
1056 	while (1) {
1057 
1058 		usb_pc_cpu_invalidate(td->page_cache);
1059 		status = hc32toh(sc, td->qtd_status);
1060 
1061 		len = EHCI_QTD_GET_BYTES(status);
1062 
1063 		/*
1064 	         * Verify the status length and
1065 		 * add the length to "frlengths[]":
1066 	         */
1067 		if (len > td->len) {
1068 			/* should not happen */
1069 			DPRINTF("Invalid status length, "
1070 			    "0x%04x/0x%04x bytes\n", len, td->len);
1071 			status |= EHCI_QTD_HALTED;
1072 		} else if (xfer->aframes != xfer->nframes) {
1073 			xfer->frlengths[xfer->aframes] += td->len - len;
1074 			/* manually update data toggle */
1075 			ehci_data_toggle_update(xfer, td->len - len, td->len);
1076 		}
1077 
1078 		/* Check for last transfer */
1079 		if (((void *)td) == xfer->td_transfer_last) {
1080 			td = NULL;
1081 			break;
1082 		}
1083 		/* Check for transfer error */
1084 		if (status & EHCI_QTD_HALTED) {
1085 			/* the transfer is finished */
1086 			td = NULL;
1087 			break;
1088 		}
1089 		/* Check for short transfer */
1090 		if (len > 0) {
1091 			if (xfer->flags_int.short_frames_ok) {
1092 				/* follow alt next */
1093 				td = td->alt_next;
1094 			} else {
1095 				/* the transfer is finished */
1096 				td = NULL;
1097 			}
1098 			break;
1099 		}
1100 		td = td->obj_next;
1101 
1102 		if (td->alt_next != td_alt_next) {
1103 			/* this USB frame is complete */
1104 			break;
1105 		}
1106 	}
1107 
1108 	/* update transfer cache */
1109 
1110 	xfer->td_transfer_cache = td;
1111 
1112 #ifdef LOSCFG_USB_DEBUG
1113 	if (status & EHCI_QTD_STATERRS) {
1114 		DPRINTFN(11, "error, addr=%d, endpt=0x%02x, frame=0x%02x"
1115 		    "status=%s%s%s%s%s%s%s%s\n",
1116 		    xfer->address, xfer->endpointno, xfer->aframes,
1117 		    (status & EHCI_QTD_ACTIVE) ? "[ACTIVE]" : "[NOT_ACTIVE]",
1118 		    (status & EHCI_QTD_HALTED) ? "[HALTED]" : "",
1119 		    (status & EHCI_QTD_BUFERR) ? "[BUFERR]" : "",
1120 		    (status & EHCI_QTD_BABBLE) ? "[BABBLE]" : "",
1121 		    (status & EHCI_QTD_XACTERR) ? "[XACTERR]" : "",
1122 		    (status & EHCI_QTD_MISSEDMICRO) ? "[MISSED]" : "",
1123 		    (status & EHCI_QTD_SPLITXSTATE) ? "[SPLIT]" : "",
1124 		    (status & EHCI_QTD_PINGSTATE) ? "[PING]" : "");
1125 	}
1126 #endif
1127 	if (status & EHCI_QTD_HALTED) {
1128 		if ((xfer->xroot->udev->parent_hs_hub != NULL) ||
1129 		    (xfer->xroot->udev->address != 0)) {
1130 			/* try to separate I/O errors from STALL */
1131 			if (EHCI_QTD_GET_CERR(status) == 0)
1132 				return (USB_ERR_IOERROR);
1133 		}
1134 		return (USB_ERR_STALLED);
1135 	}
1136 	return (USB_ERR_NORMAL_COMPLETION);
1137 }
1138 
1139 static void
ehci_non_isoc_done(struct usb_xfer * xfer)1140 ehci_non_isoc_done(struct usb_xfer *xfer)
1141 {
1142 	ehci_qh_t *qh;
1143 	usb_error_t err = USB_ERR_NORMAL_COMPLETION;
1144 
1145 	DPRINTFN(13, "xfer=%p endpoint=%p transfer done\n", xfer, xfer->endpoint);
1146 
1147 #ifdef LOSCFG_USB_DEBUG
1148 	if (ehcidebug > 10) {
1149 		ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
1150 		ehci_dump_sqtds(sc, xfer->td_transfer_first);
1151 	}
1152 #endif
1153 
1154 	/* extract data toggle directly from the QH's overlay area */
1155 	qh = (ehci_qh_t *)xfer->qh_start[xfer->flags_int.curr_dma_set];
1156 
1157 	usb_pc_cpu_invalidate(qh->page_cache);
1158 
1159 	/* reset scanner */
1160 	xfer->td_transfer_cache = xfer->td_transfer_first;
1161 	if (xfer->flags_int.control_xfr) {
1162 		if (xfer->flags_int.control_hdr) {
1163 			err = ehci_non_isoc_done_sub(xfer);
1164 		}
1165 		xfer->aframes = 1;
1166 
1167 		if (xfer->td_transfer_cache == NULL) {
1168 			goto done;
1169 		}
1170 	}
1171 	while (xfer->aframes != xfer->nframes) {
1172 
1173 		err = ehci_non_isoc_done_sub(xfer);
1174 		xfer->aframes++;
1175 
1176 		if (xfer->td_transfer_cache == NULL) {
1177 			goto done;
1178 		}
1179 	}
1180 
1181 	if (xfer->flags_int.control_xfr &&
1182 	    !xfer->flags_int.control_act) {
1183 
1184 		err = ehci_non_isoc_done_sub(xfer);
1185 	}
1186 done:
1187 	ehci_device_done(xfer, err);
1188 }
1189 
1190 /*------------------------------------------------------------------------*
1191  *	ehci_check_transfer
1192  *
1193  * Return values:
1194  *    0: USB transfer is not finished
1195  * Else: USB transfer is finished
1196  *------------------------------------------------------------------------*/
1197 static uint8_t
ehci_check_transfer(struct usb_xfer * xfer)1198 ehci_check_transfer(struct usb_xfer *xfer)
1199 {
1200 	const struct usb_pipe_methods *methods = xfer->endpoint->methods;
1201 	ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
1202 
1203 	uint32_t status;
1204 
1205 	DPRINTFN(13, "xfer=%p checking transfer\n", xfer);
1206 
1207 	if (methods == &ehci_device_isoc_fs_methods) {
1208 		ehci_sitd_t *td;
1209 
1210 		/* isochronous full speed transfer */
1211 
1212 		td = (ehci_sitd_t *)xfer->td_transfer_last;
1213 		usb_pc_cpu_invalidate(td->page_cache);
1214 		status = hc32toh(sc, td->sitd_status);
1215 
1216 		/* also check if first is complete */
1217 
1218 		td = (ehci_sitd_t *)xfer->td_transfer_first;
1219 		usb_pc_cpu_invalidate(td->page_cache);
1220 		status |= hc32toh(sc, td->sitd_status);
1221 
1222 		if (!(status & EHCI_SITD_ACTIVE)) {
1223 			ehci_device_done(xfer, USB_ERR_NORMAL_COMPLETION);
1224 			goto transferred;
1225 		}
1226 	} else if (methods == &ehci_device_isoc_hs_methods) {
1227 		ehci_itd_t *td;
1228 
1229 		/* isochronous high speed transfer */
1230 
1231 		/* check last transfer */
1232 		td = (ehci_itd_t *)xfer->td_transfer_last;
1233 		usb_pc_cpu_invalidate(td->page_cache);
1234 		status = td->itd_status[0];
1235 		status |= td->itd_status[1];
1236 		status |= td->itd_status[2];
1237 		status |= td->itd_status[3];
1238 		status |= td->itd_status[4];
1239 		status |= td->itd_status[5];
1240 		status |= td->itd_status[6];
1241 		status |= td->itd_status[7];
1242 
1243 		/* also check first transfer */
1244 		td = (ehci_itd_t *)xfer->td_transfer_first;
1245 		usb_pc_cpu_invalidate(td->page_cache);
1246 		status |= td->itd_status[0];
1247 		status |= td->itd_status[1];
1248 		status |= td->itd_status[2];
1249 		status |= td->itd_status[3];
1250 		status |= td->itd_status[4];
1251 		status |= td->itd_status[5];
1252 		status |= td->itd_status[6];
1253 		status |= td->itd_status[7];
1254 
1255 		/* if no transactions are active we continue */
1256 		if (!(status & htohc32(sc, EHCI_ITD_ACTIVE))) {
1257 			ehci_device_done(xfer, USB_ERR_NORMAL_COMPLETION);
1258 			goto transferred;
1259 		}
1260 	} else {
1261 		ehci_qtd_t *td;
1262 		ehci_qh_t *qh;
1263 
1264 		/* non-isochronous transfer */
1265 
1266 		/*
1267 		 * check whether there is an error somewhere in the middle,
1268 		 * or whether there was a short packet (SPD and not ACTIVE)
1269 		 */
1270 		td = (ehci_qtd_t *)xfer->td_transfer_cache;
1271 
1272 		qh = (ehci_qh_t *)xfer->qh_start[xfer->flags_int.curr_dma_set];
1273 
1274 		usb_pc_cpu_invalidate(qh->page_cache);
1275 
1276 		status = hc32toh(sc, qh->qh_qtd.qtd_status);
1277 		if (status & EHCI_QTD_ACTIVE) {
1278 			/* transfer is pending */
1279 			goto done;
1280 		}
1281 
1282 		while (1) {
1283 			usb_pc_cpu_invalidate(td->page_cache);
1284 			status = hc32toh(sc, td->qtd_status);
1285 
1286 			/*
1287 			 * Check if there is an active TD which
1288 			 * indicates that the transfer isn't done.
1289 			 */
1290 			if (status & EHCI_QTD_ACTIVE) {
1291 				/* update cache */
1292 				xfer->td_transfer_cache = td;
1293 				goto done;
1294 			}
1295 			/*
1296 			 * last transfer descriptor makes the transfer done
1297 			 */
1298 			if (((void *)td) == xfer->td_transfer_last) {
1299 				break;
1300 			}
1301 			/*
1302 			 * any kind of error makes the transfer done
1303 			 */
1304 			if (status & EHCI_QTD_HALTED) {
1305 				break;
1306 			}
1307 			/*
1308 			 * if there is no alternate next transfer, a short
1309 			 * packet also makes the transfer done
1310 			 */
1311 			if (EHCI_QTD_GET_BYTES(status)) {
1312 				if (xfer->flags_int.short_frames_ok) {
1313 					/* follow alt next */
1314 					if (td->alt_next) {
1315 						td = td->alt_next;
1316 						continue;
1317 					}
1318 				}
1319 				/* transfer is done */
1320 				break;
1321 			}
1322 			td = td->obj_next;
1323 		}
1324 		ehci_non_isoc_done(xfer);
1325 		goto transferred;
1326 	}
1327 
1328 done:
1329 	DPRINTFN(13, "xfer=%p is still active\n", xfer);
1330 	return (0);
1331 
1332 transferred:
1333 	return (1);
1334 }
1335 
1336 static void
ehci_pcd_enable(ehci_softc_t * sc)1337 ehci_pcd_enable(ehci_softc_t *sc)
1338 {
1339 	USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
1340 
1341 	sc->sc_eintrs |= EHCI_STS_PCD;
1342 	EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
1343 
1344 	/* acknowledge any PCD interrupt */
1345 	EOWRITE4(sc, EHCI_USBSTS, EHCI_STS_PCD);
1346 
1347 	ehci_root_intr(sc);
1348 }
1349 
1350 static void
ehci_interrupt_poll(ehci_softc_t * sc)1351 ehci_interrupt_poll(ehci_softc_t *sc)
1352 {
1353 	struct usb_xfer *xfer;
1354 
1355 repeat:
1356 	TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
1357 		/*
1358 		 * check if transfer is transferred
1359 		 */
1360 		if (ehci_check_transfer(xfer)) {
1361 			/* queue has been modified */
1362 			goto repeat;
1363 		}
1364 	}
1365 }
1366 
1367 /*
1368  * Some EHCI chips from VIA / ATI seem to trigger interrupts before
1369  * writing back the qTD status, or miss signalling occasionally under
1370  * heavy load.  If the host machine is too fast, we can miss
1371  * transaction completion - when we scan the active list the
1372  * transaction still seems to be active. This generally exhibits
1373  * itself as a umass stall that never recovers.
1374  *
1375  * We work around this behaviour by setting up this callback after any
1376  * softintr that completes with transactions still pending, giving us
1377  * another chance to check for completion after the writeback has
1378  * taken place.
1379  */
1380 static void
ehci_poll_timeout(void * ehci_arg)1381 ehci_poll_timeout(void *ehci_arg)
1382 {
1383 	ehci_softc_t *sc = (ehci_softc_t *)ehci_arg;
1384 
1385 	DPRINTFN(3, "\n");
1386 	ehci_interrupt_poll(sc);
1387 }
1388 
1389 /*------------------------------------------------------------------------*
1390  *	ehci_interrupt - EHCI interrupt handler
1391  *
1392  * NOTE: Do not access "sc->sc_bus.bdev" inside the interrupt handler,
1393  * hence the interrupt handler will be setup before "sc->sc_bus.bdev"
1394  * is present !
1395  *------------------------------------------------------------------------*/
1396 void
ehci_interrupt(unsigned int irq,ehci_softc_t * sc)1397 ehci_interrupt(unsigned int irq, ehci_softc_t *sc)
1398 {
1399 	uint32_t status;
1400 
1401 	USB_BUS_LOCK(&sc->sc_bus);
1402 	DPRINTFN(16, "real interrupt\n");
1403 
1404 #ifdef LOSCFG_USB_DEBUG
1405 	if (ehcidebug > 15) {
1406 		ehci_dump_regs(sc);
1407 	}
1408 #endif
1409 
1410 	status = EHCI_STS_INTRS(EOREAD4(sc, EHCI_USBSTS));
1411 
1412 	if (status == 0) {
1413 		/* the interrupt was not for us */
1414 		goto done;
1415 	}
1416 	if (!(status & sc->sc_eintrs)) {
1417 		goto done;
1418 	}
1419 	EOWRITE4(sc, EHCI_USBSTS, status);	/* acknowledge */
1420 
1421 	status &= sc->sc_eintrs;
1422 
1423 #ifdef LOSCFG_USB_DEBUG
1424 	if (status & EHCI_STS_HSE) {
1425 		ehci_dump_regs(sc);
1426 		ehci_dump_isoc(sc);
1427 	}
1428 #endif
1429 	if (status & EHCI_STS_PCD) {
1430 		/*
1431 		 * Disable PCD interrupt for now, because it will be
1432 		 * on until the port has been reset.
1433 		 */
1434 		sc->sc_eintrs &= ~EHCI_STS_PCD;
1435 		EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
1436 
1437 		ehci_root_intr(sc);
1438 
1439 		/* do not allow RHSC interrupts > 1 per second */
1440 		callout_reset(&sc->sc_tmo_pcd, hz,
1441 		    (void *)&ehci_pcd_enable, sc);
1442 	}
1443 	status &= ~(EHCI_STS_INT | EHCI_STS_ERRINT | EHCI_STS_PCD | EHCI_STS_IAA);
1444 
1445 	if (status != 0) {
1446 		/* block unprocessed interrupts */
1447 		sc->sc_eintrs &= ~status;
1448 		EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
1449 	}
1450 	/* poll all the USB transfers */
1451 	ehci_interrupt_poll(sc);
1452 
1453 	if (sc->sc_flags & EHCI_SCFLG_LOSTINTRBUG) {
1454 		callout_reset(&sc->sc_tmo_poll, hz / 128,
1455 		    (void *)&ehci_poll_timeout, sc);
1456 	}
1457 
1458 done:
1459 	USB_BUS_UNLOCK(&sc->sc_bus);
1460 }
1461 
1462 /*
1463  * called when a request does not complete
1464  */
1465 static void
ehci_timeout(void * ehci_arg)1466 ehci_timeout(void *ehci_arg)
1467 {
1468 	struct usb_xfer *xfer = (struct usb_xfer *)ehci_arg;
1469 
1470 	DPRINTF("xfer=%p\n", xfer);
1471 
1472 	USB_BUS_LOCK_ASSERT(xfer->xroot->bus, MA_OWNED);
1473 
1474 	/* transfer is transferred */
1475 	ehci_device_done(xfer, USB_ERR_TIMEOUT);
1476 }
1477 
1478 static void
ehci_do_poll(struct usb_bus * bus)1479 ehci_do_poll(struct usb_bus *bus)
1480 {
1481 	ehci_softc_t *sc = EHCI_BUS2SC(bus);
1482 
1483 	USB_BUS_LOCK(&sc->sc_bus);
1484 	ehci_interrupt_poll(sc);
1485 	USB_BUS_UNLOCK(&sc->sc_bus);
1486 }
1487 
1488 static void
ehci_setup_standard_chain_sub(struct ehci_std_temp * temp)1489 ehci_setup_standard_chain_sub(struct ehci_std_temp *temp)
1490 {
1491 	struct usb_page_search buf_res;
1492 	ehci_qtd_t *td;
1493 	ehci_qtd_t *td_next;
1494 	ehci_qtd_t *td_alt_next;
1495 	uint32_t buf_offset;
1496 	uint32_t average;
1497 	uint32_t len_old;
1498 	uint32_t terminate;
1499 	uint32_t qtd_altnext;
1500 	uint8_t shortpkt_old;
1501 	uint8_t precompute;
1502 
1503 	terminate = temp->sc->sc_terminate_self;
1504 	qtd_altnext = temp->sc->sc_terminate_self;
1505 	td_alt_next = NULL;
1506 	buf_offset = 0;
1507 	shortpkt_old = temp->shortpkt;
1508 	len_old = temp->len;
1509 	precompute = 1;
1510 
1511 restart:
1512 
1513 	td = temp->td;
1514 	td_next = temp->td_next;
1515 
1516 	while (1) {
1517 
1518 		if (temp->len == 0) {
1519 
1520 			if (temp->shortpkt) {
1521 				break;
1522 			}
1523 			/* send a Zero Length Packet, ZLP, last */
1524 
1525 			temp->shortpkt = 1;
1526 			average = 0;
1527 
1528 		} else {
1529 
1530 			average = temp->average;
1531 
1532 			if (temp->len < average) {
1533 				if (temp->len % temp->max_frame_size) {
1534 					temp->shortpkt = 1;
1535 				}
1536 				average = temp->len;
1537 			}
1538 		}
1539 
1540 		if (td_next == NULL) {
1541 			panic("%s: out of EHCI transfer descriptors!", __FUNCTION__);
1542 		}
1543 		/* get next TD */
1544 
1545 		td = td_next;
1546 		td_next = td->obj_next;
1547 
1548 		/* check if we are pre-computing */
1549 
1550 		if (precompute) {
1551 
1552 			/* update remaining length */
1553 
1554 			temp->len -= average;
1555 
1556 			continue;
1557 		}
1558 		/* fill out current TD */
1559 
1560 		td->qtd_status =
1561 		    temp->qtd_status |
1562 		    htohc32(temp->sc, EHCI_QTD_IOC |
1563 		    EHCI_QTD_SET_BYTES(average));
1564 
1565 		if (average == 0) {
1566 
1567 			if (temp->auto_data_toggle == 0) {
1568 
1569 				/* update data toggle, ZLP case */
1570 
1571 				temp->qtd_status ^=
1572 				    htohc32(temp->sc, EHCI_QTD_TOGGLE_MASK);
1573 			}
1574 			td->len = 0;
1575 
1576 			/* properly reset reserved fields */
1577 			td->qtd_buffer[0] = 0;
1578 			td->qtd_buffer[1] = 0;
1579 			td->qtd_buffer[2] = 0;
1580 			td->qtd_buffer[3] = 0;
1581 			td->qtd_buffer[4] = 0;
1582 			td->qtd_buffer_hi[0] = 0;
1583 			td->qtd_buffer_hi[1] = 0;
1584 			td->qtd_buffer_hi[2] = 0;
1585 			td->qtd_buffer_hi[3] = 0;
1586 			td->qtd_buffer_hi[4] = 0;
1587 		} else {
1588 
1589 			uint8_t x;
1590 
1591 			if (temp->auto_data_toggle == 0) {
1592 
1593 				/* update data toggle */
1594 
1595 				if (((average + temp->max_frame_size - 1) /
1596 				    temp->max_frame_size) & 1) {
1597 					temp->qtd_status ^=
1598 					    htohc32(temp->sc, EHCI_QTD_TOGGLE_MASK);
1599 				}
1600 			}
1601 			td->len = average;
1602 
1603 			/* update remaining length */
1604 
1605 			temp->len -= average;
1606 
1607 			/* fill out buffer pointers */
1608 
1609 			usbd_get_page(temp->pc, buf_offset, &buf_res);
1610 			td->qtd_buffer[0] =
1611 #if USB_HAVE_BUSDMA
1612 			    htohc32(temp->sc, buf_res.physaddr);
1613 #else
1614 			    htohc32(temp->sc, (unsigned int)buf_res.buffer);
1615 #endif
1616 			td->qtd_buffer_hi[0] = 0;
1617 
1618 			x = 1;
1619 
1620 			while (average > EHCI_PAGE_SIZE) {
1621 				average -= EHCI_PAGE_SIZE;
1622 				buf_offset += EHCI_PAGE_SIZE;
1623 				usbd_get_page(temp->pc, buf_offset, &buf_res);
1624 				td->qtd_buffer[x] =
1625 				    htohc32(temp->sc,
1626 #if USB_HAVE_BUSDMA
1627 				    buf_res.physaddr & (~0xFFF));
1628 #else
1629 				    (unsigned int)buf_res.buffer & (~0xFFF));
1630 #endif
1631 				td->qtd_buffer_hi[x] = 0;
1632 				x++;
1633 			}
1634 
1635 			/*
1636 			 * NOTE: The "average" variable is never zero after
1637 			 * exiting the loop above !
1638 			 *
1639 			 * NOTE: We have to subtract one from the offset to
1640 			 * ensure that we are computing the physical address
1641 			 * of a valid page !
1642 			 */
1643 			buf_offset += average;
1644 			usbd_get_page(temp->pc, buf_offset - 1, &buf_res);
1645 			td->qtd_buffer[x] =
1646 			    htohc32(temp->sc,
1647 #if USB_HAVE_BUSDMA
1648 			    buf_res.physaddr & (~0xFFF));
1649 #else
1650 			    (unsigned int)buf_res.buffer & (~0xFFF));
1651 #endif
1652 			td->qtd_buffer_hi[x] = 0;
1653 
1654 			/* properly reset reserved fields */
1655 			while (++x < EHCI_QTD_NBUFFERS) {
1656 				td->qtd_buffer[x] = 0;
1657 				td->qtd_buffer_hi[x] = 0;
1658 			}
1659 		}
1660 
1661 		if (td_next) {
1662 
1663 			/* link the current TD with the next one */
1664 			td->qtd_next = td_next->qtd_self;
1665 		}
1666 
1667 		td->qtd_altnext = qtd_altnext;
1668 		td->alt_next = td_alt_next;
1669 
1670 		usb_pc_cpu_flush(td->page_cache);
1671 	}
1672 
1673 	if (precompute) {
1674 		precompute = 0;
1675 
1676 		/* setup alt next pointer, if any */
1677 		if (temp->last_frame) {
1678 			td_alt_next = NULL;
1679 			qtd_altnext = terminate;
1680 		} else {
1681 			/* we use this field internally */
1682 			td_alt_next = td_next;
1683 			if (temp->setup_alt_next && td_next) {
1684 				qtd_altnext = td_next->qtd_self;
1685 			} else {
1686 				qtd_altnext = terminate;
1687 			}
1688 		}
1689 
1690 		/* restore */
1691 		temp->shortpkt = shortpkt_old;
1692 		temp->len = len_old;
1693 		goto restart;
1694 	}
1695 	temp->td = td;
1696 	temp->td_next = td_next;
1697 }
1698 
1699 static void
ehci_setup_standard_chain(struct usb_xfer * xfer,ehci_qh_t ** qh_last)1700 ehci_setup_standard_chain(struct usb_xfer *xfer, ehci_qh_t **qh_last)
1701 {
1702 	struct ehci_std_temp temp;
1703 	const struct usb_pipe_methods *methods;
1704 	ehci_qh_t *qh;
1705 	ehci_qtd_t *td;
1706 	uint32_t qh_endp;
1707 	uint32_t qh_endphub;
1708 	uint32_t x;
1709 	uint32_t int_save;
1710 
1711 	DPRINTFN(9, "addr=%d endpt=%d sumlen=%d speed=%d\n",
1712 	    xfer->address, UE_GET_ADDR(xfer->endpointno),
1713 	    xfer->sumlen, usbd_get_speed(xfer->xroot->udev));
1714 
1715 	temp.average = xfer->max_hc_frame_size;
1716 	temp.max_frame_size = xfer->max_frame_size;
1717 	temp.sc = EHCI_BUS2SC(xfer->xroot->bus);
1718 
1719 	/* toggle the DMA set we are using */
1720 	xfer->flags_int.curr_dma_set ^= 1;
1721 
1722 	/* get next DMA set */
1723 	td = (ehci_qtd_t *)xfer->td_start[xfer->flags_int.curr_dma_set];
1724 
1725 	xfer->td_transfer_first = td;
1726 	xfer->td_transfer_cache = td;
1727 
1728 	temp.td = NULL;
1729 	temp.td_next = td;
1730 	temp.qtd_status = 0;
1731 	temp.last_frame = 0;
1732 	temp.setup_alt_next = xfer->flags_int.short_frames_ok;
1733 
1734 	if (xfer->flags_int.control_xfr) {
1735 		if (xfer->endpoint->toggle_next) {
1736 			/* DATA1 is next */
1737 			temp.qtd_status |=
1738 			    htohc32(temp.sc, EHCI_QTD_SET_TOGGLE(1U));
1739 		}
1740 		temp.auto_data_toggle = 0;
1741 	} else {
1742 		temp.auto_data_toggle = 1;
1743 	}
1744 
1745 	if ((xfer->xroot->udev->parent_hs_hub != NULL) ||
1746 	    (xfer->xroot->udev->address != 0)) {
1747 		/* max 3 retries */
1748 		temp.qtd_status |=
1749 		    htohc32(temp.sc, EHCI_QTD_SET_CERR(3));
1750 	}
1751 	/* check if we should prepend a setup message */
1752 
1753 	if (xfer->flags_int.control_xfr) {
1754 		if (xfer->flags_int.control_hdr) {
1755 
1756 			xfer->endpoint->toggle_next = 0;
1757 
1758 			temp.qtd_status &=
1759 			    htohc32(temp.sc, EHCI_QTD_SET_CERR(3));
1760 			temp.qtd_status |= htohc32(temp.sc,
1761 			    EHCI_QTD_ACTIVE |
1762 			    EHCI_QTD_SET_PID(EHCI_QTD_PID_SETUP) |
1763 			    EHCI_QTD_SET_TOGGLE(0));
1764 
1765 			temp.len = xfer->frlengths[0];
1766 			temp.pc = xfer->frbuffers + 0;
1767 			temp.shortpkt = temp.len ? 1 : 0;
1768 			/* check for last frame */
1769 			if (xfer->nframes == 1) {
1770 				/* no STATUS stage yet, SETUP is last */
1771 				if (xfer->flags_int.control_act) {
1772 					temp.last_frame = 1;
1773 					temp.setup_alt_next = 0;
1774 				}
1775 			}
1776 			ehci_setup_standard_chain_sub(&temp);
1777 		}
1778 		x = 1;
1779 	} else {
1780 		x = 0;
1781 	}
1782 
1783 	while (x != xfer->nframes) {
1784 
1785 		/* DATA0 / DATA1 message */
1786 
1787 		temp.len = xfer->frlengths[x];
1788 		temp.pc = xfer->frbuffers + x;
1789 
1790 		x++;
1791 
1792 		if (x == xfer->nframes) {
1793 			if (xfer->flags_int.control_xfr) {
1794 				/* no STATUS stage yet, DATA is last */
1795 				if (xfer->flags_int.control_act) {
1796 					temp.last_frame = 1;
1797 					temp.setup_alt_next = 0;
1798 				}
1799 			} else {
1800 				temp.last_frame = 1;
1801 				temp.setup_alt_next = 0;
1802 			}
1803 		}
1804 		/* keep previous data toggle and error count */
1805 
1806 		temp.qtd_status &=
1807 		    htohc32(temp.sc, EHCI_QTD_SET_CERR(3) |
1808 		    EHCI_QTD_SET_TOGGLE(1U));
1809 
1810 		if (temp.len == 0) {
1811 
1812 			/* make sure that we send an USB packet */
1813 
1814 			temp.shortpkt = 0;
1815 
1816 		} else {
1817 
1818 			/* regular data transfer */
1819 
1820 			temp.shortpkt = (xfer->flags.force_short_xfer) ? 0 : 1;
1821 		}
1822 
1823 		/* set endpoint direction */
1824 
1825 		temp.qtd_status |=
1826 		    (UE_GET_DIR(xfer->endpointno) == UE_DIR_IN) ?
1827 		    htohc32(temp.sc, EHCI_QTD_ACTIVE |
1828 		    EHCI_QTD_SET_PID(EHCI_QTD_PID_IN)) :
1829 		    htohc32(temp.sc, EHCI_QTD_ACTIVE |
1830 		    EHCI_QTD_SET_PID(EHCI_QTD_PID_OUT));
1831 
1832 		ehci_setup_standard_chain_sub(&temp);
1833 	}
1834 
1835 	/* check if we should append a status stage */
1836 
1837 	if (xfer->flags_int.control_xfr &&
1838 	    !xfer->flags_int.control_act) {
1839 
1840 		/*
1841 		 * Send a DATA1 message and invert the current endpoint
1842 		 * direction.
1843 		 */
1844 
1845 		temp.qtd_status &= htohc32(temp.sc, EHCI_QTD_SET_CERR(3) |
1846 		    EHCI_QTD_SET_TOGGLE(1U));
1847 		temp.qtd_status |=
1848 		    (UE_GET_DIR(xfer->endpointno) == UE_DIR_OUT) ?
1849 		    htohc32(temp.sc, EHCI_QTD_ACTIVE |
1850 		    EHCI_QTD_SET_PID(EHCI_QTD_PID_IN) |
1851 		    EHCI_QTD_SET_TOGGLE(1U)) :
1852 		    htohc32(temp.sc, EHCI_QTD_ACTIVE |
1853 		    EHCI_QTD_SET_PID(EHCI_QTD_PID_OUT) |
1854 		    EHCI_QTD_SET_TOGGLE(1U));
1855 
1856 		temp.len = 0;
1857 		temp.pc = NULL;
1858 		temp.shortpkt = 0;
1859 		temp.last_frame = 1;
1860 		temp.setup_alt_next = 0;
1861 
1862 		ehci_setup_standard_chain_sub(&temp);
1863 	}
1864 	td = temp.td;
1865 	if (td == NULL)
1866 		return;
1867 
1868 	/* the last TD terminates the transfer: */
1869 	td->qtd_next = htohc32(temp.sc, EHCI_LINK_TERMINATE);
1870 	td->qtd_altnext = htohc32(temp.sc, EHCI_LINK_TERMINATE);
1871 
1872 	usb_pc_cpu_flush(td->page_cache);
1873 
1874 	/* must have at least one frame! */
1875 
1876 	xfer->td_transfer_last = td;
1877 
1878 #ifdef LOSCFG_USB_DEBUG
1879 	if (ehcidebug > 8) {
1880 		DPRINTF("nexttog=%d; data before transfer:\n",
1881 		    xfer->endpoint->toggle_next);
1882 		ehci_dump_sqtds(temp.sc,
1883 		    xfer->td_transfer_first);
1884 	}
1885 #endif
1886 
1887 	methods = xfer->endpoint->methods;
1888 
1889 	qh = (ehci_qh_t *)xfer->qh_start[xfer->flags_int.curr_dma_set];
1890 
1891 	/* the "qh_link" field is filled when the QH is added */
1892 
1893 	qh_endp =
1894 	    (EHCI_QH_SET_ADDR(xfer->address) |
1895 	    EHCI_QH_SET_ENDPT(UE_GET_ADDR(xfer->endpointno)) |
1896 	    EHCI_QH_SET_MPL(xfer->max_packet_size));
1897 
1898 	if (usbd_get_speed(xfer->xroot->udev) == USB_SPEED_HIGH) {
1899 		qh_endp |= EHCI_QH_SET_EPS(EHCI_QH_SPEED_HIGH);
1900 		if (methods != &ehci_device_intr_methods)
1901 			qh_endp |= EHCI_QH_SET_NRL(8U);
1902 	} else {
1903 
1904 		if (usbd_get_speed(xfer->xroot->udev) == USB_SPEED_FULL) {
1905 			qh_endp |= EHCI_QH_SET_EPS(EHCI_QH_SPEED_FULL);
1906 		} else {
1907 			qh_endp |= EHCI_QH_SET_EPS(EHCI_QH_SPEED_LOW);
1908 		}
1909 
1910 		if (methods == &ehci_device_ctrl_methods) {
1911 			qh_endp |= EHCI_QH_CTL;
1912 		}
1913 		if (methods != &ehci_device_intr_methods) {
1914 			/* Only try one time per microframe! */
1915 			qh_endp |= EHCI_QH_SET_NRL(1);
1916 		}
1917 	}
1918 
1919 	if (temp.auto_data_toggle == 0) {
1920 		/* software computes the data toggle */
1921 		qh_endp |= EHCI_QH_DTC;
1922 	}
1923 
1924 	qh->qh_endp = htohc32(temp.sc, qh_endp);
1925 
1926 	qh_endphub =
1927 	    (EHCI_QH_SET_MULT(xfer->max_packet_count & 3) |
1928 	    EHCI_QH_SET_CMASK(xfer->endpoint->usb_cmask) |
1929 	    EHCI_QH_SET_SMASK(xfer->endpoint->usb_smask) |
1930 	    EHCI_QH_SET_HUBA(xfer->xroot->udev->hs_hub_addr) |
1931 	    EHCI_QH_SET_PORT(xfer->xroot->udev->hs_port_no));
1932 
1933 	qh->qh_endphub = htohc32(temp.sc, qh_endphub);
1934 	qh->qh_curqtd = 0;
1935 
1936 	/* fill the overlay qTD */
1937 
1938 	if (temp.auto_data_toggle && xfer->endpoint->toggle_next) {
1939 		/* DATA1 is next */
1940 		qh->qh_qtd.qtd_status = htohc32(temp.sc, EHCI_QTD_SET_TOGGLE(1U));
1941 	} else {
1942 		qh->qh_qtd.qtd_status = 0;
1943 	}
1944 
1945 	td = (ehci_qtd_t *)xfer->td_transfer_first;
1946 
1947 	qh->qh_qtd.qtd_next = td->qtd_self;
1948 	qh->qh_qtd.qtd_altnext =
1949 	    htohc32(temp.sc, EHCI_LINK_TERMINATE);
1950 
1951 	/* properly reset reserved fields */
1952 	qh->qh_qtd.qtd_buffer[0] = 0;
1953 	qh->qh_qtd.qtd_buffer[1] = 0;
1954 	qh->qh_qtd.qtd_buffer[2] = 0;
1955 	qh->qh_qtd.qtd_buffer[3] = 0;
1956 	qh->qh_qtd.qtd_buffer[4] = 0;
1957 	qh->qh_qtd.qtd_buffer_hi[0] = 0;
1958 	qh->qh_qtd.qtd_buffer_hi[1] = 0;
1959 	qh->qh_qtd.qtd_buffer_hi[2] = 0;
1960 	qh->qh_qtd.qtd_buffer_hi[3] = 0;
1961 	qh->qh_qtd.qtd_buffer_hi[4] = 0;
1962 
1963 	usb_pc_cpu_flush(qh->page_cache);
1964 
1965 	if (xfer->xroot->udev->flags.self_suspended == 0) {
1966 		spin_lock_irqsave(&g_usb_ehci_qh_spinlock, int_save);
1967 		EHCI_APPEND_QH(qh, *qh_last);
1968 		spin_unlock_irqrestore(&g_usb_ehci_qh_spinlock, int_save);
1969 	}
1970 }
1971 
1972 static void
ehci_root_intr(ehci_softc_t * sc)1973 ehci_root_intr(ehci_softc_t *sc)
1974 {
1975 	uint16_t i;
1976 	uint16_t m;
1977 	USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
1978 
1979 	/* clear any old interrupt data */
1980 	(void)memset_s(sc->sc_hub_idata, sizeof(sc->sc_hub_idata), 0, sizeof(sc->sc_hub_idata));
1981 
1982 	/* set bits */
1983 	m = (sc->sc_noport + 1);
1984 	if (m > (8 * sizeof(sc->sc_hub_idata))) {
1985 		m = (8 * sizeof(sc->sc_hub_idata));
1986 	}
1987 	for (i = 1; i < m; i++) {
1988 		/* pick out CHANGE bits from the status register */
1989 		if (EOREAD4(sc, EHCI_PORTSC(i)) & EHCI_PS_CLEAR) {
1990 			sc->sc_hub_idata[i / 8] |= 1 << (i % 8);
1991 			DPRINTF("port %d changed\n", i);
1992 		}
1993 	}
1994 	uhub_root_intr(&sc->sc_bus, sc->sc_hub_idata,
1995 	    sizeof(sc->sc_hub_idata));
1996 }
1997 
1998 static void
ehci_isoc_fs_done(ehci_softc_t * sc,struct usb_xfer * xfer)1999 ehci_isoc_fs_done(ehci_softc_t *sc, struct usb_xfer *xfer)
2000 {
2001 	uint32_t nframes = xfer->nframes;
2002 	uint32_t status;
2003 	uint32_t *plen = xfer->frlengths;
2004 	uint16_t len = 0;
2005 	ehci_sitd_t *td = (ehci_sitd_t *)xfer->td_transfer_first;
2006 	ehci_sitd_t **pp_last = &sc->sc_isoc_fs_p_last[xfer->qh_pos];
2007 
2008 	DPRINTFN(13, "xfer=%p endpoint=%p transfer done\n",
2009 	    xfer, xfer->endpoint);
2010 
2011 	while (nframes--) {
2012 		if (td == NULL) {
2013 			panic("%s:%d: out of TD's\n",
2014 			    __FUNCTION__, __LINE__);
2015 		}
2016 		if (pp_last >= &sc->sc_isoc_fs_p_last[EHCI_VIRTUAL_FRAMELIST_COUNT]) {
2017 			pp_last = &sc->sc_isoc_fs_p_last[0];
2018 		}
2019 #ifdef LOSCFG_USB_DEBUG
2020 		if (ehcidebug > 15) {
2021 			DPRINTF("isoc FS-TD\n");
2022 			ehci_dump_sitd(sc, td);
2023 		}
2024 #endif
2025 		usb_pc_cpu_invalidate(td->page_cache);
2026 		status = hc32toh(sc, td->sitd_status);
2027 
2028 		len = EHCI_SITD_GET_LEN(status);
2029 
2030 		DPRINTFN(2, "status=0x%08x, rem=%u\n", status, len);
2031 
2032 		if (*plen >= len) {
2033 			len = *plen - len;
2034 		} else {
2035 			len = 0;
2036 		}
2037 
2038 		*plen = len;
2039 
2040 		/* remove FS-TD from schedule */
2041 		EHCI_REMOVE_FS_TD(td, *pp_last);
2042 
2043 		pp_last++;
2044 		plen++;
2045 		td = td->obj_next;
2046 	}
2047 
2048 	xfer->aframes = xfer->nframes;
2049 }
2050 
2051 static void
ehci_isoc_hs_done(ehci_softc_t * sc,struct usb_xfer * xfer)2052 ehci_isoc_hs_done(ehci_softc_t *sc, struct usb_xfer *xfer)
2053 {
2054 	uint32_t nframes = xfer->nframes;
2055 	uint32_t status;
2056 	uint32_t *plen = xfer->frlengths;
2057 	uint16_t len = 0;
2058 	uint8_t td_no = 0;
2059 	ehci_itd_t *td = (ehci_itd_t *)xfer->td_transfer_first;
2060 	ehci_itd_t **pp_last = &sc->sc_isoc_hs_p_last[xfer->qh_pos];
2061 
2062 	DPRINTFN(13, "xfer=%p endpoint=%p transfer done\n",
2063 	    xfer, xfer->endpoint);
2064 
2065 	while (nframes) {
2066 		if (td == NULL) {
2067 			panic("%s:%d: out of TD's\n",
2068 			    __FUNCTION__, __LINE__);
2069 		}
2070 		if (pp_last >= &sc->sc_isoc_hs_p_last[EHCI_VIRTUAL_FRAMELIST_COUNT]) {
2071 			pp_last = &sc->sc_isoc_hs_p_last[0];
2072 		}
2073 #ifdef LOSCFG_USB_DEBUG
2074 		if (ehcidebug > 15) {
2075 			DPRINTF("isoc HS-TD\n");
2076 			ehci_dump_itd(sc, td);
2077 		}
2078 #endif
2079 
2080 		usb_pc_cpu_invalidate(td->page_cache);
2081 		status = hc32toh(sc, td->itd_status[td_no]);
2082 
2083 		len = EHCI_ITD_GET_LEN(status);
2084 
2085 		DPRINTFN(2, "status=0x%08x, len=%u\n", status, len);
2086 
2087 		if (xfer->endpoint->usb_smask & (1 << td_no)) {
2088 
2089 			if (*plen >= len) {
2090 				/*
2091 				 * The length is valid. NOTE: The
2092 				 * complete length is written back
2093 				 * into the status field, and not the
2094 				 * remainder like with other transfer
2095 				 * descriptor types.
2096 				 */
2097 			} else {
2098 				/* Invalid length - truncate */
2099 				len = 0;
2100 			}
2101 
2102 			*plen = len;
2103 			plen++;
2104 			nframes--;
2105 		}
2106 
2107 		td_no++;
2108 
2109 		if ((td_no == 8) || (nframes == 0)) {
2110 			/* remove HS-TD from schedule */
2111 			EHCI_REMOVE_HS_TD(td, *pp_last);
2112 			pp_last++;
2113 
2114 			td_no = 0;
2115 			td = td->obj_next;
2116 		}
2117 	}
2118 	xfer->aframes = xfer->nframes;
2119 }
2120 
2121 /* NOTE: "done" can be run two times in a row,
2122  * from close and from interrupt
2123  */
2124 static void
ehci_device_done(struct usb_xfer * xfer,usb_error_t error)2125 ehci_device_done(struct usb_xfer *xfer, usb_error_t error)
2126 {
2127 	uintptr_t interrupt_ret;
2128 	const struct usb_pipe_methods *methods = xfer->endpoint->methods;
2129 	ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2130 
2131 	USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
2132 
2133 	DPRINTFN(2, "xfer=%p, endpoint=%p, error=%d\n",
2134 	    xfer, xfer->endpoint, error);
2135 
2136 	if ((methods == &ehci_device_bulk_methods) ||
2137 	    (methods == &ehci_device_ctrl_methods)) {
2138 #ifdef LOSCFG_USB_DEBUG
2139 		if (ehcidebug > 8) {
2140 			DPRINTF("nexttog=%d; data after transfer:\n",
2141 			    xfer->endpoint->toggle_next);
2142 			ehci_dump_sqtds(sc,
2143 			    xfer->td_transfer_first);
2144 		}
2145 #endif
2146 
2147 		spin_lock_irqsave(&g_usb_ehci_qh_spinlock, interrupt_ret);
2148 		EHCI_REMOVE_QH((ehci_qh_t *)xfer->qh_start[xfer->flags_int.curr_dma_set],
2149 		    sc->sc_async_p_last);
2150 		spin_unlock_irqrestore(&g_usb_ehci_qh_spinlock, interrupt_ret);
2151 	}
2152 	if (methods == &ehci_device_intr_methods) {
2153 		spin_lock_irqsave(&g_usb_ehci_qh_spinlock, interrupt_ret);
2154 		EHCI_REMOVE_QH((ehci_qh_t *)xfer->qh_start[xfer->flags_int.curr_dma_set],
2155 		    sc->sc_intr_p_last[xfer->qh_pos]);
2156 		spin_unlock_irqrestore(&g_usb_ehci_qh_spinlock, interrupt_ret);
2157 	}
2158 	/*
2159 	 * Only finish isochronous transfers once which will update
2160 	 * "xfer->frlengths".
2161 	 */
2162 	if (xfer->td_transfer_first &&
2163 	    xfer->td_transfer_last) {
2164 		if (methods == &ehci_device_isoc_fs_methods) {
2165 			ehci_isoc_fs_done(sc, xfer);
2166 		}
2167 		if (methods == &ehci_device_isoc_hs_methods) {
2168 			ehci_isoc_hs_done(sc, xfer);
2169 		}
2170 		xfer->td_transfer_first = NULL;
2171 		xfer->td_transfer_last = NULL;
2172 	}
2173 	/* dequeue transfer and start next transfer */
2174 	usbd_transfer_done(xfer, error);
2175 }
2176 
2177 /*------------------------------------------------------------------------*
2178  * ehci bulk support
2179  *------------------------------------------------------------------------*/
2180 static void
ehci_device_bulk_open(struct usb_xfer * xfer)2181 ehci_device_bulk_open(struct usb_xfer *xfer)
2182 {
2183 	return;
2184 }
2185 
2186 static void
ehci_device_bulk_close(struct usb_xfer * xfer)2187 ehci_device_bulk_close(struct usb_xfer *xfer)
2188 {
2189 	ehci_device_done(xfer, USB_ERR_CANCELLED);
2190 }
2191 
2192 static void
ehci_device_bulk_enter(struct usb_xfer * xfer)2193 ehci_device_bulk_enter(struct usb_xfer *xfer)
2194 {
2195 	return;
2196 }
2197 
2198 static void
ehci_doorbell_async(struct ehci_softc * sc)2199 ehci_doorbell_async(struct ehci_softc *sc)
2200 {
2201 	uint32_t temp;
2202 
2203 	/*
2204 	 * XXX Performance quirk: Some Host Controllers have a too low
2205 	 * interrupt rate. Issue an IAAD to stimulate the Host
2206 	 * Controller after queueing the BULK transfer.
2207 	 *
2208 	 * XXX Force the host controller to refresh any QH caches.
2209 	 */
2210 	temp = EOREAD4(sc, EHCI_USBCMD);
2211 	if (!(temp & EHCI_CMD_IAAD))
2212 		EOWRITE4(sc, EHCI_USBCMD, temp | EHCI_CMD_IAAD);
2213 }
2214 
2215 static void
ehci_device_bulk_start(struct usb_xfer * xfer)2216 ehci_device_bulk_start(struct usb_xfer *xfer)
2217 {
2218 	ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2219 
2220 	/* setup TD's and QH */
2221 	ehci_setup_standard_chain(xfer, &sc->sc_async_p_last);
2222 
2223 	/* put transfer on interrupt queue */
2224 	ehci_transfer_intr_enqueue(xfer);
2225 
2226 	/*
2227 	 * XXX Certain nVidia chipsets choke when using the IAAD
2228 	 * feature too frequently.
2229 	 */
2230 	if (sc->sc_flags & EHCI_SCFLG_IAADBUG)
2231 		return;
2232 
2233 	ehci_doorbell_async(sc);
2234 }
2235 
2236 const struct usb_pipe_methods ehci_device_bulk_methods =
2237 {
2238 	.open = ehci_device_bulk_open,
2239 	.close = ehci_device_bulk_close,
2240 	.enter = ehci_device_bulk_enter,
2241 	.start = ehci_device_bulk_start,
2242 };
2243 
2244 /*------------------------------------------------------------------------*
2245  * ehci control support
2246  *------------------------------------------------------------------------*/
2247 static void
ehci_device_ctrl_open(struct usb_xfer * xfer)2248 ehci_device_ctrl_open(struct usb_xfer *xfer)
2249 {
2250 	return;
2251 }
2252 
2253 static void
ehci_device_ctrl_close(struct usb_xfer * xfer)2254 ehci_device_ctrl_close(struct usb_xfer *xfer)
2255 {
2256 	ehci_device_done(xfer, USB_ERR_CANCELLED);
2257 }
2258 
2259 static void
ehci_device_ctrl_enter(struct usb_xfer * xfer)2260 ehci_device_ctrl_enter(struct usb_xfer *xfer)
2261 {
2262 	return;
2263 }
2264 
2265 static void
ehci_device_ctrl_start(struct usb_xfer * xfer)2266 ehci_device_ctrl_start(struct usb_xfer *xfer)
2267 {
2268 	ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2269 
2270 	/* setup TD's and QH */
2271 	ehci_setup_standard_chain(xfer, &sc->sc_async_p_last);
2272 
2273 	/* put transfer on interrupt queue */
2274 	ehci_transfer_intr_enqueue(xfer);
2275 }
2276 
2277 const struct usb_pipe_methods ehci_device_ctrl_methods =
2278 {
2279 	.open = ehci_device_ctrl_open,
2280 	.close = ehci_device_ctrl_close,
2281 	.enter = ehci_device_ctrl_enter,
2282 	.start = ehci_device_ctrl_start,
2283 };
2284 
2285 /*------------------------------------------------------------------------*
2286  * ehci interrupt support
2287  *------------------------------------------------------------------------*/
2288 static void
ehci_device_intr_open(struct usb_xfer * xfer)2289 ehci_device_intr_open(struct usb_xfer *xfer)
2290 {
2291 	ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2292 	uint16_t best;
2293 	uint16_t bit;
2294 	uint16_t x;
2295 
2296 	usb_hs_bandwidth_alloc(xfer);
2297 
2298 	/*
2299 	 * Find the best QH position corresponding to the given interval:
2300 	 */
2301 
2302 	best = 0;
2303 	bit = EHCI_VIRTUAL_FRAMELIST_COUNT / 2;
2304 	while (bit) {
2305 		if (xfer->interval >= bit) {
2306 			x = bit;
2307 			best = bit;
2308 			while (x & bit) {
2309 				if (sc->sc_intr_stat[x] <
2310 				    sc->sc_intr_stat[best]) {
2311 					best = x;
2312 				}
2313 				x++;
2314 			}
2315 			break;
2316 		}
2317 		bit >>= 1;
2318 	}
2319 
2320 	sc->sc_intr_stat[best]++;
2321 	xfer->qh_pos = best;
2322 
2323 	DPRINTFN(3, "best=%d interval=%d\n",
2324 	    best, xfer->interval);
2325 }
2326 
2327 static void
ehci_device_intr_close(struct usb_xfer * xfer)2328 ehci_device_intr_close(struct usb_xfer *xfer)
2329 {
2330 	ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2331 
2332 	sc->sc_intr_stat[xfer->qh_pos]--;
2333 
2334 	ehci_device_done(xfer, USB_ERR_CANCELLED);
2335 
2336 	/* bandwidth must be freed after device done */
2337 	usb_hs_bandwidth_free(xfer);
2338 }
2339 
2340 static void
ehci_device_intr_enter(struct usb_xfer * xfer)2341 ehci_device_intr_enter(struct usb_xfer *xfer)
2342 {
2343 	return;
2344 }
2345 
2346 static void
ehci_device_intr_start(struct usb_xfer * xfer)2347 ehci_device_intr_start(struct usb_xfer *xfer)
2348 {
2349 	ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2350 
2351 	/* setup TD's and QH */
2352 	ehci_setup_standard_chain(xfer, &sc->sc_intr_p_last[xfer->qh_pos]);
2353 
2354 	/* put transfer on interrupt queue */
2355 	ehci_transfer_intr_enqueue(xfer);
2356 }
2357 
2358 const struct usb_pipe_methods ehci_device_intr_methods =
2359 {
2360 	.open = ehci_device_intr_open,
2361 	.close = ehci_device_intr_close,
2362 	.enter = ehci_device_intr_enter,
2363 	.start = ehci_device_intr_start,
2364 };
2365 
2366 /*------------------------------------------------------------------------*
2367  * ehci full speed isochronous support
2368  *------------------------------------------------------------------------*/
2369 static void
ehci_device_isoc_fs_open(struct usb_xfer * xfer)2370 ehci_device_isoc_fs_open(struct usb_xfer *xfer)
2371 {
2372 	ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2373 	ehci_sitd_t *td;
2374 	uint32_t sitd_portaddr;
2375 	uint8_t ds;
2376 
2377 	sitd_portaddr =
2378 	    EHCI_SITD_SET_ADDR(xfer->address) |
2379 	    EHCI_SITD_SET_ENDPT(UE_GET_ADDR(xfer->endpointno)) |
2380 	    EHCI_SITD_SET_HUBA(xfer->xroot->udev->hs_hub_addr) |
2381 	    EHCI_SITD_SET_PORT(xfer->xroot->udev->hs_port_no);
2382 
2383 	if (UE_GET_DIR(xfer->endpointno) == UE_DIR_IN)
2384 		sitd_portaddr |= EHCI_SITD_SET_DIR_IN;
2385 
2386 	sitd_portaddr = htohc32(sc, sitd_portaddr);
2387 
2388 	/* initialize all TD's */
2389 
2390 	for (ds = 0; ds != 2; ds++) {
2391 
2392 		for (td = (ehci_sitd_t *)xfer->td_start[ds]; td; td = td->obj_next) {
2393 
2394 			td->sitd_portaddr = sitd_portaddr;
2395 
2396 			/*
2397 			 * TODO: make some kind of automatic
2398 			 * SMASK/CMASK selection based on micro-frame
2399 			 * usage
2400 			 *
2401 			 * micro-frame usage (8 microframes per 1ms)
2402 			 */
2403 			td->sitd_back = htohc32(sc, EHCI_LINK_TERMINATE);
2404 
2405 			usb_pc_cpu_flush(td->page_cache);
2406 		}
2407 	}
2408 }
2409 
2410 static void
ehci_device_isoc_fs_close(struct usb_xfer * xfer)2411 ehci_device_isoc_fs_close(struct usb_xfer *xfer)
2412 {
2413 	ehci_device_done(xfer, USB_ERR_CANCELLED);
2414 }
2415 
2416 static void
ehci_device_isoc_fs_enter(struct usb_xfer * xfer)2417 ehci_device_isoc_fs_enter(struct usb_xfer *xfer)
2418 {
2419 	struct usb_page_search buf_res;
2420 	ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2421 	ehci_sitd_t *td;
2422 	ehci_sitd_t *td_last = NULL;
2423 	ehci_sitd_t **pp_last;
2424 	uint32_t *plen;
2425 	uint32_t buf_offset;
2426 	uint32_t nframes;
2427 	uint32_t temp;
2428 	uint32_t sitd_mask;
2429 	uint16_t tlen;
2430 	uint8_t sa;
2431 	uint8_t sb;
2432 
2433 #ifdef LOSCFG_USB_DEBUG
2434 	uint8_t once = 1;
2435 #endif
2436 
2437 	DPRINTFN(6, "xfer=%p next=%d nframes=%d\n",
2438 	    xfer, xfer->endpoint->isoc_next, xfer->nframes);
2439 
2440 	/* get the current frame index */
2441 
2442 	nframes = EOREAD4(sc, EHCI_FRINDEX) / 8;
2443 
2444 	/*
2445 	 * check if the frame index is within the window where the frames
2446 	 * will be inserted
2447 	 */
2448 	buf_offset = (nframes - xfer->endpoint->isoc_next) &
2449 	    (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
2450 
2451 	if ((xfer->endpoint->is_synced == 0) ||
2452 	    (buf_offset < xfer->nframes)) {
2453 		/*
2454 		 * If there is data underflow or the pipe queue is empty we
2455 		 * schedule the transfer a few frames ahead of the current
2456 		 * frame position. Else two isochronous transfers might
2457 		 * overlap.
2458 		 */
2459 		xfer->endpoint->isoc_next = (nframes + 3) &
2460 		    (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
2461 		xfer->endpoint->is_synced = 1;
2462 		DPRINTFN(3, "start next=%d\n", xfer->endpoint->isoc_next);
2463 	}
2464 	/*
2465 	 * compute how many milliseconds the insertion is ahead of the
2466 	 * current frame position:
2467 	 */
2468 	buf_offset = (xfer->endpoint->isoc_next - nframes) &
2469 	    (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
2470 
2471 	/*
2472 	 * pre-compute when the isochronous transfer will be finished:
2473 	 */
2474 	xfer->isoc_time_complete =
2475 	    usb_isoc_time_expand(&sc->sc_bus, nframes) +
2476 	    buf_offset + xfer->nframes;
2477 
2478 	/* get the real number of frames */
2479 
2480 	nframes = xfer->nframes;
2481 
2482 	buf_offset = 0;
2483 
2484 	plen = xfer->frlengths;
2485 
2486 	/* toggle the DMA set we are using */
2487 	xfer->flags_int.curr_dma_set ^= 1;
2488 
2489 	/* get next DMA set */
2490 	td = (ehci_sitd_t *)xfer->td_start[xfer->flags_int.curr_dma_set];
2491 	xfer->td_transfer_first = td;
2492 
2493 	pp_last = &sc->sc_isoc_fs_p_last[xfer->endpoint->isoc_next];
2494 
2495 	/* store starting position */
2496 
2497 	xfer->qh_pos = xfer->endpoint->isoc_next;
2498 
2499 	while (nframes--) {
2500 		if (td == NULL) {
2501 			panic("%s:%d: out of TD's\n",
2502 			    __FUNCTION__, __LINE__);
2503 		}
2504 		if (pp_last >= &sc->sc_isoc_fs_p_last[EHCI_VIRTUAL_FRAMELIST_COUNT])
2505 			pp_last = &sc->sc_isoc_fs_p_last[0];
2506 
2507 		/* reuse sitd_portaddr and sitd_back from last transfer */
2508 
2509 		if (*plen > xfer->max_frame_size) {
2510 #ifdef LOSCFG_USB_DEBUG
2511 			if (once) {
2512 				once = 0;
2513 				PRINTK("%s: frame length(%d) exceeds %d "
2514 				    "bytes (frame truncated)\n",
2515 				    __FUNCTION__, *plen,
2516 				    xfer->max_frame_size);
2517 			}
2518 #endif
2519 			*plen = xfer->max_frame_size;
2520 		}
2521 
2522 		/* allocate a slot */
2523 
2524 		sa = usbd_fs_isoc_schedule_alloc_slot(xfer,
2525 		    xfer->isoc_time_complete - nframes - 1);
2526 
2527 		if (sa == 255) {
2528 			/*
2529 			 * Schedule is FULL, set length to zero:
2530 			 */
2531 
2532 			*plen = 0;
2533 			sa = USB_FS_ISOC_UFRAME_MAX - 1;
2534 		}
2535 		if (*plen) {
2536 			/*
2537 			 * only call "usbd_get_page()" when we have a
2538 			 * non-zero length
2539 			 */
2540 			usbd_get_page(xfer->frbuffers, buf_offset, &buf_res);
2541 #if USB_HAVE_BUSDMA
2542 			td->sitd_bp[0] = htohc32(sc, buf_res.physaddr);
2543 #else
2544 			td->sitd_bp[0] = htohc32(sc, (unsigned int)buf_res.buffer);
2545 #endif
2546 			buf_offset += *plen;
2547 			/*
2548 			 * NOTE: We need to subtract one from the offset so
2549 			 * that we are on a valid page!
2550 			 */
2551 			usbd_get_page(xfer->frbuffers, buf_offset - 1,
2552 			    &buf_res);
2553 #if USB_HAVE_BUSDMA
2554 			temp = buf_res.physaddr & ~0xFFF;
2555 #else
2556 			temp = (unsigned int)buf_res.buffer & ~0xFFF;
2557 #endif
2558 		} else {
2559 			td->sitd_bp[0] = 0;
2560 			temp = 0;
2561 		}
2562 
2563 		if (UE_GET_DIR(xfer->endpointno) == UE_DIR_OUT) {
2564 			tlen = *plen;
2565 			if (tlen <= 188) {
2566 				temp |= 1;	/* T-count = 1, TP = ALL */
2567 				tlen = 1;
2568 			} else {
2569 				tlen += 187;
2570 				tlen /= 188;
2571 				temp |= (uint32_t)tlen;	/* T-count = [1..6] */
2572 				temp |= 8;	/* TP = Begin */
2573 			}
2574 
2575 			tlen += sa;
2576 
2577 			if (tlen >= 8) {
2578 				sb = 0;
2579 			} else {
2580 				sb = (1 << tlen);
2581 			}
2582 
2583 			sa = (1 << sa);
2584 			sa = (sb - sa) & 0x3F;
2585 			sb = 0;
2586 		} else {
2587 			sb = (-(4 << sa)) & 0xFE;
2588 			sa = (1 << sa) & 0x3F;
2589 		}
2590 
2591 		sitd_mask = (EHCI_SITD_SET_SMASK(sa) |
2592 		    EHCI_SITD_SET_CMASK(sb));
2593 
2594 		td->sitd_bp[1] = htohc32(sc, temp);
2595 
2596 		td->sitd_mask = htohc32(sc, sitd_mask);
2597 
2598 		if (nframes == 0) {
2599 			td->sitd_status = htohc32(sc,
2600 			    EHCI_SITD_IOC |
2601 			    EHCI_SITD_ACTIVE |
2602 			    EHCI_SITD_SET_LEN(*plen));
2603 		} else {
2604 			td->sitd_status = htohc32(sc,
2605 			    EHCI_SITD_ACTIVE |
2606 			    EHCI_SITD_SET_LEN(*plen));
2607 		}
2608 		usb_pc_cpu_flush(td->page_cache);
2609 
2610 #ifdef LOSCFG_USB_DEBUG
2611 		if (ehcidebug > 15) {
2612 			DPRINTF("FS-TD %d\n", nframes);
2613 			ehci_dump_sitd(sc, td);
2614 		}
2615 #endif
2616 		/* insert TD into schedule */
2617 		EHCI_APPEND_FS_TD(td, *pp_last);
2618 		pp_last++;
2619 
2620 		plen++;
2621 		td_last = td;
2622 		td = td->obj_next;
2623 	}
2624 
2625 	xfer->td_transfer_last = td_last;
2626 
2627 	/* update isoc_next */
2628 	xfer->endpoint->isoc_next = (pp_last - &sc->sc_isoc_fs_p_last[0]) &
2629 	    (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
2630 
2631 	/*
2632 	 * We don't allow cancelling of the SPLIT transaction USB FULL
2633 	 * speed transfer, because it disturbs the bandwidth
2634 	 * computation algorithm.
2635 	 */
2636 	xfer->flags_int.can_cancel_immed = 0;
2637 }
2638 
2639 static void
ehci_device_isoc_fs_start(struct usb_xfer * xfer)2640 ehci_device_isoc_fs_start(struct usb_xfer *xfer)
2641 {
2642 	/*
2643 	 * We don't allow cancelling of the SPLIT transaction USB FULL
2644 	 * speed transfer, because it disturbs the bandwidth
2645 	 * computation algorithm.
2646 	 */
2647 	xfer->flags_int.can_cancel_immed = 0;
2648 
2649 	/* set a default timeout */
2650 	if (xfer->timeout == 0)
2651 		xfer->timeout = 500; /* ms */
2652 
2653 	/* put transfer on interrupt queue */
2654 	ehci_transfer_intr_enqueue(xfer);
2655 }
2656 
2657 const struct usb_pipe_methods ehci_device_isoc_fs_methods = {
2658 	.open = ehci_device_isoc_fs_open,
2659 	.close = ehci_device_isoc_fs_close,
2660 	.enter = ehci_device_isoc_fs_enter,
2661 	.start = ehci_device_isoc_fs_start,
2662 };
2663 
2664 /*------------------------------------------------------------------------*
2665  * ehci high speed isochronous support
2666  *------------------------------------------------------------------------*/
2667 static void
ehci_device_isoc_hs_open(struct usb_xfer * xfer)2668 ehci_device_isoc_hs_open(struct usb_xfer *xfer)
2669 {
2670 	ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2671 	ehci_itd_t *td;
2672 	uint32_t temp;
2673 	uint8_t ds;
2674 
2675 	usb_hs_bandwidth_alloc(xfer);
2676 
2677 	/* initialize all TD's */
2678 
2679 	for (ds = 0; ds != 2; ds++) {
2680 
2681 		for (td = (ehci_itd_t *)xfer->td_start[ds]; td; td = td->obj_next) {
2682 
2683 			/* set TD inactive */
2684 			td->itd_status[0] = 0;
2685 			td->itd_status[1] = 0;
2686 			td->itd_status[2] = 0;
2687 			td->itd_status[3] = 0;
2688 			td->itd_status[4] = 0;
2689 			td->itd_status[5] = 0;
2690 			td->itd_status[6] = 0;
2691 			td->itd_status[7] = 0;
2692 
2693 			/* set endpoint and address */
2694 			td->itd_bp[0] = htohc32(sc,
2695 			    EHCI_ITD_SET_ADDR(xfer->address) |
2696 			    EHCI_ITD_SET_ENDPT(UE_GET_ADDR(xfer->endpointno)));
2697 
2698 			temp =
2699 			    EHCI_ITD_SET_MPL(xfer->max_packet_size & 0x7FF);
2700 
2701 			/* set direction */
2702 			if (UE_GET_DIR(xfer->endpointno) == UE_DIR_IN) {
2703 				temp |= EHCI_ITD_SET_DIR_IN;
2704 			}
2705 			/* set maximum packet size */
2706 			td->itd_bp[1] = htohc32(sc, temp);
2707 
2708 			/* set transfer multiplier */
2709 			td->itd_bp[2] = htohc32(sc, xfer->max_packet_count & 3);
2710 
2711 			usb_pc_cpu_flush(td->page_cache);
2712 		}
2713 	}
2714 }
2715 
2716 static void
ehci_device_isoc_hs_close(struct usb_xfer * xfer)2717 ehci_device_isoc_hs_close(struct usb_xfer *xfer)
2718 {
2719 	ehci_device_done(xfer, USB_ERR_CANCELLED);
2720 
2721 	/* bandwidth must be freed after device done */
2722 	usb_hs_bandwidth_free(xfer);
2723 }
2724 
2725 static void
ehci_device_isoc_hs_enter(struct usb_xfer * xfer)2726 ehci_device_isoc_hs_enter(struct usb_xfer *xfer)
2727 {
2728 	struct usb_page_search buf_res;
2729 	ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2730 	ehci_itd_t *td;
2731 	ehci_itd_t *td_last = NULL;
2732 	ehci_itd_t **pp_last;
2733 	bus_size_t page_addr;
2734 	uint32_t *plen;
2735 	uint32_t status;
2736 	uint32_t buf_offset;
2737 	uint32_t nframes;
2738 	uint32_t itd_offset[8 + 1];
2739 	uint8_t x;
2740 	uint8_t td_no;
2741 	uint8_t page_no;
2742 	uint8_t shift = usbd_xfer_get_fps_shift(xfer);
2743 
2744 #ifdef LOSCFG_USB_DEBUG
2745 	uint8_t once = 1;
2746 #endif
2747 
2748 	DPRINTFN(6, "xfer=%p next=%d nframes=%d shift=%d\n",
2749 	    xfer, xfer->endpoint->isoc_next, xfer->nframes, (int)shift);
2750 
2751 	/* get the current frame index */
2752 
2753 	nframes = EOREAD4(sc, EHCI_FRINDEX) / 8;
2754 
2755 	/*
2756 	 * check if the frame index is within the window where the frames
2757 	 * will be inserted
2758 	 */
2759 	buf_offset = (nframes - xfer->endpoint->isoc_next) &
2760 	    (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
2761 
2762 	if ((xfer->endpoint->is_synced == 0) ||
2763 	    (buf_offset < (((xfer->nframes << shift) + 7) / 8))) {
2764 		/*
2765 		 * If there is data underflow or the pipe queue is empty we
2766 		 * schedule the transfer a few frames ahead of the current
2767 		 * frame position. Else two isochronous transfers might
2768 		 * overlap.
2769 		 */
2770 		xfer->endpoint->isoc_next = (nframes + 3) &
2771 		    (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
2772 		xfer->endpoint->is_synced = 1;
2773 		DPRINTFN(3, "start next=%d\n", xfer->endpoint->isoc_next);
2774 	}
2775 	/*
2776 	 * compute how many milliseconds the insertion is ahead of the
2777 	 * current frame position:
2778 	 */
2779 	buf_offset = (xfer->endpoint->isoc_next - nframes) &
2780 	    (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
2781 
2782 	/*
2783 	 * pre-compute when the isochronous transfer will be finished:
2784 	 */
2785 	xfer->isoc_time_complete =
2786 	    usb_isoc_time_expand(&sc->sc_bus, nframes) + buf_offset +
2787 	    (((xfer->nframes << shift) + 7) / 8);
2788 
2789 	/* get the real number of frames */
2790 
2791 	nframes = xfer->nframes;
2792 
2793 	buf_offset = 0;
2794 	td_no = 0;
2795 
2796 	plen = xfer->frlengths;
2797 
2798 	/* toggle the DMA set we are using */
2799 	xfer->flags_int.curr_dma_set ^= 1;
2800 
2801 	/* get next DMA set */
2802 	td = (ehci_itd_t *)xfer->td_start[xfer->flags_int.curr_dma_set];
2803 	xfer->td_transfer_first = td;
2804 
2805 	pp_last = &sc->sc_isoc_hs_p_last[xfer->endpoint->isoc_next];
2806 
2807 	/* store starting position */
2808 
2809 	xfer->qh_pos = xfer->endpoint->isoc_next;
2810 
2811 	while (nframes) {
2812 		if (td == NULL) {
2813 			panic("%s:%d: out of TD's\n",
2814 			    __FUNCTION__, __LINE__);
2815 		}
2816 		if (pp_last >= &sc->sc_isoc_hs_p_last[EHCI_VIRTUAL_FRAMELIST_COUNT]) {
2817 			pp_last = &sc->sc_isoc_hs_p_last[0];
2818 		}
2819 		/* range check */
2820 		if (*plen > xfer->max_frame_size) {
2821 #ifdef LOSCFG_USB_DEBUG
2822 			if (once) {
2823 				once = 0;
2824 				PRINTK("%s: frame length(%d) exceeds %d bytes "
2825 				    "(frame truncated)\n",
2826 				    __FUNCTION__, *plen, xfer->max_frame_size);
2827 			}
2828 #endif
2829 			*plen = xfer->max_frame_size;
2830 		}
2831 
2832 		if (xfer->endpoint->usb_smask & (1 << td_no)) {
2833 			status = (EHCI_ITD_SET_LEN(*plen) |
2834 			    EHCI_ITD_ACTIVE |
2835 			    EHCI_ITD_SET_PG(0));
2836 			td->itd_status[td_no] = htohc32(sc, status);
2837 			itd_offset[td_no] = buf_offset;
2838 			buf_offset += *plen;
2839 			plen++;
2840 			nframes --;
2841 		} else {
2842 			td->itd_status[td_no] = 0;	/* not active */
2843 			itd_offset[td_no] = buf_offset;
2844 		}
2845 
2846 		td_no++;
2847 
2848 		if ((td_no == 8) || (nframes == 0)) {
2849 
2850 			/* the rest of the transfers are not active, if any */
2851 			for (x = td_no; x != 8; x++) {
2852 				td->itd_status[x] = 0;	/* not active */
2853 			}
2854 
2855 			/* check if there is any data to be transferred */
2856 			if (itd_offset[0] != buf_offset) {
2857 				page_no = 0;
2858 				itd_offset[td_no] = buf_offset;
2859 
2860 				/* get first page offset */
2861 				usbd_get_page(xfer->frbuffers, itd_offset[0], &buf_res);
2862 				/* get page address */
2863 #if USB_HAVE_BUSDMA
2864 				page_addr = buf_res.physaddr & ~0xFFF;
2865 #else
2866 				page_addr = (unsigned int)buf_res.buffer & ~0xFFF;
2867 #endif
2868 				/* update page address */
2869 				td->itd_bp[0] &= htohc32(sc, 0xFFF);
2870 				td->itd_bp[0] |= htohc32(sc, page_addr);
2871 
2872 				for (x = 0; x != td_no; x++) {
2873 					/* set page number and page offset */
2874 					status = (EHCI_ITD_SET_PG(page_no) |
2875 #if USB_HAVE_BUSDMA
2876 					    (buf_res.physaddr & 0xFFF));
2877 #else
2878 					((unsigned int)buf_res.buffer & 0xFFF));
2879 #endif
2880 					td->itd_status[x] |= htohc32(sc, status);
2881 
2882 					/* get next page offset */
2883 					if (itd_offset[x + 1] == buf_offset) {
2884 						/*
2885 						 * We subtract one so that
2886 						 * we don't go off the last
2887 						 * page!
2888 						 */
2889 						usbd_get_page(xfer->frbuffers, buf_offset - 1, &buf_res);
2890 					} else {
2891 						usbd_get_page(xfer->frbuffers, itd_offset[x + 1], &buf_res);
2892 					}
2893 #if USB_HAVE_BUSDMA
2894 					/* check if we need a new page */
2895 					if ((buf_res.physaddr ^ page_addr) & ~0xFFF) {
2896 						/* new page needed */
2897 						page_addr = buf_res.physaddr & ~0xFFF;
2898 #else
2899 					/* check if we need a new page */
2900 					if (((unsigned int)buf_res.buffer ^ page_addr) & ~0xFFF) {
2901 						/* new page needed */
2902 						page_addr = (unsigned int)buf_res.buffer & ~0xFFF;
2903 #endif
2904 						if (page_no == 6) {
2905 							panic("%s: too many pages\n", __FUNCTION__);
2906 						}
2907 						page_no++;
2908 						page_no%= EHCI_ITD_BP_MAX;
2909 						/* update page address */
2910 						td->itd_bp[page_no] &= htohc32(sc, 0xFFF);
2911 						td->itd_bp[page_no] |= htohc32(sc, page_addr);
2912 					}
2913 				}
2914 			}
2915 			/* set IOC bit if we are complete */
2916 			if (nframes == 0) {
2917 				td->itd_status[td_no - 1] |= htohc32(sc, EHCI_ITD_IOC);
2918 			}
2919 			usb_pc_cpu_flush(td->page_cache);
2920 #ifdef LOSCFG_USB_DEBUG
2921 			if (ehcidebug > 15) {
2922 				DPRINTF("HS-TD %d\n", nframes);
2923 				ehci_dump_itd(sc, td);
2924 			}
2925 #endif
2926 			/* insert TD into schedule */
2927 			EHCI_APPEND_HS_TD(td, *pp_last);
2928 			pp_last++;
2929 
2930 			td_no = 0;
2931 			td_last = td;
2932 			td = td->obj_next;
2933 		}
2934 	}
2935 
2936 	xfer->td_transfer_last = td_last;
2937 
2938 	/* update isoc_next */
2939 	xfer->endpoint->isoc_next = (pp_last - &sc->sc_isoc_hs_p_last[0]) &
2940 	    (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
2941 }
2942 
2943 static void
2944 ehci_device_isoc_hs_start(struct usb_xfer *xfer)
2945 {
2946 	/* put transfer on interrupt queue */
2947 	ehci_transfer_intr_enqueue(xfer);
2948 }
2949 
2950 const struct usb_pipe_methods ehci_device_isoc_hs_methods = {
2951 	.open = ehci_device_isoc_hs_open,
2952 	.close = ehci_device_isoc_hs_close,
2953 	.enter = ehci_device_isoc_hs_enter,
2954 	.start = ehci_device_isoc_hs_start,
2955 };
2956 
2957 /*------------------------------------------------------------------------*
2958  * ehci root control support
2959  *------------------------------------------------------------------------*
2960  * Simulate a hardware hub by handling all the necessary requests.
2961  *------------------------------------------------------------------------*/
2962 
2963 static const
2964 struct usb_device_descriptor ehci_devd = {
2965 	sizeof(struct usb_device_descriptor),
2966 	UDESC_DEVICE,			/* type */
2967 	{0x00, 0x02},			/* USB version */
2968 	UDCLASS_HUB,			/* class */
2969 	UDSUBCLASS_HUB,			/* subclass */
2970 	UDPROTO_HSHUBSTT,		/* protocol */
2971 	64,				/* max packet */
2972 	{0}, {0}, {0x00, 0x01},		/* device id */
2973 	1, 2, 0,			/* string indicies */
2974 	1				/* # of configurations */
2975 };
2976 
2977 static const
2978 struct usb_device_qualifier ehci_odevd = {
2979 	sizeof(struct usb_device_qualifier),
2980 	UDESC_DEVICE_QUALIFIER,		/* type */
2981 	{0x00, 0x02},			/* USB version */
2982 	UDCLASS_HUB,			/* class */
2983 	UDSUBCLASS_HUB,			/* subclass */
2984 	UDPROTO_FSHUB,			/* protocol */
2985 	0,				/* max packet */
2986 	0,				/* # of configurations */
2987 	0
2988 };
2989 
2990 static const struct ehci_config_desc ehci_confd = {
2991 	.confd = {
2992 		.bLength = sizeof(struct usb_config_descriptor),
2993 		.bDescriptorType = UDESC_CONFIG,
2994 		.wTotalLength[0] = sizeof(ehci_confd),
2995 		.bNumInterface = 1,
2996 		.bConfigurationValue = 1,
2997 		.iConfiguration = 0,
2998 		.bmAttributes = UC_SELF_POWERED,
2999 		.bMaxPower = 0		/* max power */
3000 	},
3001 	.ifcd = {
3002 		.bLength = sizeof(struct usb_interface_descriptor),
3003 		.bDescriptorType = UDESC_INTERFACE,
3004 		.bNumEndpoints = 1,
3005 		.bInterfaceClass = UICLASS_HUB,
3006 		.bInterfaceSubClass = UISUBCLASS_HUB,
3007 		.bInterfaceProtocol = 0,
3008 	},
3009 	.endpd = {
3010 		.bLength = sizeof(struct usb_endpoint_descriptor),
3011 		.bDescriptorType = UDESC_ENDPOINT,
3012 		.bEndpointAddress = UE_DIR_IN | EHCI_INTR_ENDPT,
3013 		.bmAttributes = UE_INTERRUPT,
3014 		.wMaxPacketSize[0] = 8,	/* max packet (63 ports) */
3015 		.bInterval = 255,
3016 	},
3017 };
3018 
3019 static const
3020 struct usb_hub_descriptor ehci_hubd = {
3021 	.bDescLength = 0,		/* dynamic length */
3022 	.bDescriptorType = UDESC_HUB,
3023 };
3024 
3025 uint16_t
3026 ehci_get_port_speed_portsc(struct ehci_softc *sc, uint16_t index)
3027 {
3028 	uint32_t v;
3029 
3030 	v = EOREAD4(sc, EHCI_PORTSC(index));
3031 	v = (v >> EHCI_PORTSC_PSPD_SHIFT) & EHCI_PORTSC_PSPD_MASK;
3032 
3033 	if (v == EHCI_PORT_SPEED_HIGH)
3034 		return (UPS_HIGH_SPEED);
3035 	if (v == EHCI_PORT_SPEED_LOW)
3036 		return (UPS_LOW_SPEED);
3037 	return (0);
3038 }
3039 
3040 uint16_t
3041 ehci_get_port_speed_hostc(struct ehci_softc *sc, uint16_t index)
3042 {
3043 	uint32_t v;
3044 
3045 	v = EOREAD4(sc, EHCI_HOSTC(index));
3046 	v = (v >> EHCI_HOSTC_PSPD_SHIFT) & EHCI_HOSTC_PSPD_MASK;
3047 
3048 	if (v == EHCI_PORT_SPEED_HIGH)
3049 		return (UPS_HIGH_SPEED);
3050 	if (v == EHCI_PORT_SPEED_LOW)
3051 		return (UPS_LOW_SPEED);
3052 	return (0);
3053 }
3054 
3055 static usb_error_t
3056 ehci_roothub_exec(struct usb_device *udev,
3057     struct usb_device_request *req, const void **pptr, uint16_t *plength)
3058 {
3059 	ehci_softc_t *sc = EHCI_BUS2SC(udev->bus);
3060 	const char *str_ptr;
3061 	const void *ptr;
3062 	uint32_t port;
3063 	uint32_t v;
3064 	uint16_t len;
3065 	uint16_t i;
3066 	uint16_t value;
3067 	uint16_t index;
3068 	usb_error_t err = USB_ERR_NORMAL_COMPLETION;
3069 
3070 	USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
3071 
3072 	/* buffer reset */
3073 	ptr = (const void *)&sc->sc_hub_desc;
3074 	len = 0;
3075 
3076 	value = UGETW(req->wValue);
3077 	index = UGETW(req->wIndex);
3078 
3079 	DPRINTFN(3, "type=0x%02x request=0x%02x wLen=0x%04x "
3080 	    "wValue=0x%04x wIndex=0x%04x\n",
3081 	    req->bmRequestType, req->bRequest,
3082 	    UGETW(req->wLength), value, index);
3083 
3084 #define	C(x,y) ((x) | ((y) << 8))
3085 	switch (C(req->bRequest, req->bmRequestType)) {
3086 	case C(UR_CLEAR_FEATURE, UT_WRITE_DEVICE):
3087 	case C(UR_CLEAR_FEATURE, UT_WRITE_INTERFACE):
3088 	case C(UR_CLEAR_FEATURE, UT_WRITE_ENDPOINT):
3089 		/*
3090 		 * DEVICE_REMOTE_WAKEUP and ENDPOINT_HALT are no-ops
3091 		 * for the integrated root hub.
3092 		 */
3093 		break;
3094 	case C(UR_GET_CONFIG, UT_READ_DEVICE):
3095 		len = 1;
3096 		sc->sc_hub_desc.temp[0] = sc->sc_conf;
3097 		break;
3098 	case C(UR_GET_DESCRIPTOR, UT_READ_DEVICE):
3099 		switch (value >> 8) {
3100 		case UDESC_DEVICE:
3101 			if ((value & 0xff) != 0) {
3102 				err = USB_ERR_IOERROR;
3103 				goto done;
3104 			}
3105 			len = sizeof(ehci_devd);
3106 			ptr = (const void *)&ehci_devd;
3107 			break;
3108 			/*
3109 			 * We can't really operate at another speed,
3110 			 * but the specification says we need this
3111 			 * descriptor:
3112 			 */
3113 		case UDESC_DEVICE_QUALIFIER:
3114 			if ((value & 0xff) != 0) {
3115 				err = USB_ERR_IOERROR;
3116 				goto done;
3117 			}
3118 			len = sizeof(ehci_odevd);
3119 			ptr = (const void *)&ehci_odevd;
3120 			break;
3121 
3122 		case UDESC_CONFIG:
3123 			if ((value & 0xff) != 0) {
3124 				err = USB_ERR_IOERROR;
3125 				goto done;
3126 			}
3127 			len = sizeof(ehci_confd);
3128 			ptr = (const void *)&ehci_confd;
3129 			break;
3130 
3131 		case UDESC_STRING:
3132 			switch (value & 0xff) {
3133 			case 0:	/* Language table */
3134 				str_ptr = "\001";
3135 				break;
3136 
3137 			case 1:	/* Vendor */
3138 				str_ptr = sc->sc_vendor;
3139 				break;
3140 
3141 			case 2:	/* Product */
3142 				str_ptr = "EHCI root HUB";
3143 				break;
3144 
3145 			default:
3146 				str_ptr = "";
3147 				break;
3148 			}
3149 
3150 			len = usb_make_str_desc(
3151 			    sc->sc_hub_desc.temp,
3152 			    sizeof(sc->sc_hub_desc.temp),
3153 			    str_ptr);
3154 			break;
3155 		default:
3156 			err = USB_ERR_IOERROR;
3157 			goto done;
3158 		}
3159 		break;
3160 	case C(UR_GET_INTERFACE, UT_READ_INTERFACE):
3161 		len = 1;
3162 		sc->sc_hub_desc.temp[0] = 0;
3163 		break;
3164 	case C(UR_GET_STATUS, UT_READ_DEVICE):
3165 		len = 2;
3166 		USETW(sc->sc_hub_desc.stat.wStatus, UDS_SELF_POWERED);
3167 		break;
3168 	case C(UR_GET_STATUS, UT_READ_INTERFACE):
3169 	case C(UR_GET_STATUS, UT_READ_ENDPOINT):
3170 		len = 2;
3171 		USETW(sc->sc_hub_desc.stat.wStatus, 0);
3172 		break;
3173 	case C(UR_SET_ADDRESS, UT_WRITE_DEVICE):
3174 		if (value >= EHCI_MAX_DEVICES) {
3175 			err = USB_ERR_IOERROR;
3176 			goto done;
3177 		}
3178 		sc->sc_addr = value;
3179 		break;
3180 	case C(UR_SET_CONFIG, UT_WRITE_DEVICE):
3181 		if ((value != 0) && (value != 1)) {
3182 			err = USB_ERR_IOERROR;
3183 			goto done;
3184 		}
3185 		sc->sc_conf = value;
3186 		break;
3187 	case C(UR_SET_DESCRIPTOR, UT_WRITE_DEVICE):
3188 		break;
3189 	case C(UR_SET_FEATURE, UT_WRITE_DEVICE):
3190 	case C(UR_SET_FEATURE, UT_WRITE_INTERFACE):
3191 	case C(UR_SET_FEATURE, UT_WRITE_ENDPOINT):
3192 		err = USB_ERR_IOERROR;
3193 		goto done;
3194 	case C(UR_SET_INTERFACE, UT_WRITE_INTERFACE):
3195 		break;
3196 	case C(UR_SYNCH_FRAME, UT_WRITE_ENDPOINT):
3197 		break;
3198 		/* Hub requests */
3199 	case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_DEVICE):
3200 		break;
3201 	case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_OTHER):
3202 		DPRINTFN(9, "UR_CLEAR_PORT_FEATURE\n");
3203 
3204 		if ((index < 1) ||
3205 		    (index > sc->sc_noport)) {
3206 			err = USB_ERR_IOERROR;
3207 			goto done;
3208 		}
3209 		port = EHCI_PORTSC(index);
3210 		v = EOREAD4(sc, port) & ~EHCI_PS_CLEAR;
3211 		switch (value) {
3212 		case UHF_PORT_ENABLE:
3213 			EOWRITE4(sc, port, v & ~EHCI_PS_PE);
3214 			break;
3215 		case UHF_PORT_SUSPEND:
3216 			if ((v & EHCI_PS_SUSP) && (!(v & EHCI_PS_FPR))) {
3217 
3218 				/*
3219 				 * waking up a High Speed device is rather
3220 				 * complicated if
3221 				 */
3222 				EOWRITE4(sc, port, v | EHCI_PS_FPR);
3223 			}
3224 			/* wait 20ms for resume sequence to complete */
3225 			usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 50);
3226 
3227 			EOWRITE4(sc, port, v & ~(EHCI_PS_SUSP |
3228 			    EHCI_PS_FPR | (3 << 10) /* High Speed */ ));
3229 
3230 			/* 4ms settle time */
3231 			usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 250);
3232 			break;
3233 		case UHF_PORT_POWER:
3234 			EOWRITE4(sc, port, v & ~EHCI_PS_PP);
3235 			break;
3236 		case UHF_PORT_TEST:
3237 			DPRINTFN(3, "clear port test "
3238 			    "%d\n", index);
3239 			break;
3240 		case UHF_PORT_INDICATOR:
3241 			DPRINTFN(3, "clear port ind "
3242 			    "%d\n", index);
3243 			EOWRITE4(sc, port, v & ~EHCI_PS_PIC);
3244 			break;
3245 		case UHF_C_PORT_CONNECTION:
3246 			EOWRITE4(sc, port, v | EHCI_PS_CSC);
3247 			break;
3248 		case UHF_C_PORT_ENABLE:
3249 			EOWRITE4(sc, port, v | EHCI_PS_PEC);
3250 			break;
3251 		case UHF_C_PORT_SUSPEND:
3252 			EOWRITE4(sc, port, v | EHCI_PS_SUSP);
3253 			break;
3254 		case UHF_C_PORT_OVER_CURRENT:
3255 			EOWRITE4(sc, port, v | EHCI_PS_OCC);
3256 			break;
3257 		case UHF_C_PORT_RESET:
3258 			sc->sc_isreset = 0;
3259 			break;
3260 		default:
3261 			err = USB_ERR_IOERROR;
3262 			goto done;
3263 		}
3264 		break;
3265 	case C(UR_GET_DESCRIPTOR, UT_READ_CLASS_DEVICE):
3266 		if ((value & 0xff) != 0) {
3267 			err = USB_ERR_IOERROR;
3268 			goto done;
3269 		}
3270 		v = EREAD4(sc, EHCI_HCSPARAMS);
3271 
3272 		sc->sc_hub_desc.hubd = ehci_hubd;
3273 		sc->sc_hub_desc.hubd.bNbrPorts = sc->sc_noport;
3274 
3275 		if (EHCI_HCS_PPC(v))
3276 			i = UHD_PWR_INDIVIDUAL;
3277 		else
3278 			i = UHD_PWR_NO_SWITCH;
3279 
3280 		if (EHCI_HCS_P_INDICATOR(v))
3281 			i |= UHD_PORT_IND;
3282 
3283 		USETW(sc->sc_hub_desc.hubd.wHubCharacteristics, i);
3284 		/* XXX can't find out? */
3285 		sc->sc_hub_desc.hubd.bPwrOn2PwrGood = 200;
3286 		/* XXX don't know if ports are removable or not */
3287 		sc->sc_hub_desc.hubd.bDescLength =
3288 		    8 + ((sc->sc_noport + 7) / 8);
3289 		len = sc->sc_hub_desc.hubd.bDescLength;
3290 		break;
3291 	case C(UR_GET_STATUS, UT_READ_CLASS_DEVICE):
3292 		len = 16;
3293 		(void)memset_s(sc->sc_hub_desc.temp, sizeof(sc->sc_hub_desc.temp), 0, len);
3294 		break;
3295 	case C(UR_GET_STATUS, UT_READ_CLASS_OTHER):
3296 		DPRINTFN(9, "get port status i=%d\n",
3297 		    index);
3298 		if ((index < 1) ||
3299 		    (index > sc->sc_noport)) {
3300 			err = USB_ERR_IOERROR;
3301 			goto done;
3302 		}
3303 		v = EOREAD4(sc, EHCI_PORTSC(index));
3304 		DPRINTFN(1, "port status=0x%04x\n", v);
3305 		if (sc->sc_flags & EHCI_SCFLG_TT) {
3306 			if (sc->sc_vendor_get_port_speed != NULL) {
3307 				i = sc->sc_vendor_get_port_speed(sc, index);
3308 			} else {
3309 				device_printf(sc->sc_bus.bdev,
3310 				    "EHCI_SCFLG_TT quirk is set but "
3311 				    "sc_vendor_get_hub_speed() is NULL\n");
3312 				i = UPS_HIGH_SPEED;
3313 			}
3314 		} else {
3315 			i = UPS_HIGH_SPEED;
3316 		}
3317 		if (v & EHCI_PS_CS)
3318 			i |= UPS_CURRENT_CONNECT_STATUS;
3319 		if (v & EHCI_PS_PE)
3320 			i |= UPS_PORT_ENABLED;
3321 		if ((v & EHCI_PS_SUSP) && !(v & EHCI_PS_FPR))
3322 			i |= UPS_SUSPEND;
3323 		if (v & EHCI_PS_OCA)
3324 			i |= UPS_OVERCURRENT_INDICATOR;
3325 		if (v & EHCI_PS_PR)
3326 			i |= UPS_RESET;
3327 		if (v & EHCI_PS_PP)
3328 			i |= UPS_PORT_POWER;
3329 		USETW(sc->sc_hub_desc.ps.wPortStatus, i);
3330 		i = 0;
3331 		if (v & EHCI_PS_CSC)
3332 			i |= UPS_C_CONNECT_STATUS;
3333 		if (v & EHCI_PS_PEC)
3334 			i |= UPS_C_PORT_ENABLED;
3335 		if (v & EHCI_PS_OCC)
3336 			i |= UPS_C_OVERCURRENT_INDICATOR;
3337 		if (v & EHCI_PS_FPR)
3338 			i |= UPS_C_SUSPEND;
3339 		if (sc->sc_isreset)
3340 			i |= UPS_C_PORT_RESET;
3341 		USETW(sc->sc_hub_desc.ps.wPortChange, i);
3342 		len = sizeof(sc->sc_hub_desc.ps);
3343 		break;
3344 	case C(UR_SET_DESCRIPTOR, UT_WRITE_CLASS_DEVICE):
3345 		err = USB_ERR_IOERROR;
3346 		goto done;
3347 	case C(UR_SET_FEATURE, UT_WRITE_CLASS_DEVICE):
3348 		break;
3349 	case C(UR_SET_FEATURE, UT_WRITE_CLASS_OTHER):
3350 		if ((index < 1) ||
3351 		    (index > sc->sc_noport)) {
3352 			err = USB_ERR_IOERROR;
3353 			goto done;
3354 		}
3355 		port = EHCI_PORTSC(index);
3356 		v = EOREAD4(sc, port) & ~EHCI_PS_CLEAR;
3357 		switch (value) {
3358 		case UHF_PORT_ENABLE:
3359 			EOWRITE4(sc, port, v | EHCI_PS_PE);
3360 			break;
3361 		case UHF_PORT_SUSPEND:
3362 			EOWRITE4(sc, port, v | EHCI_PS_SUSP);
3363 			break;
3364 		case UHF_PORT_RESET:
3365 			DPRINTFN(6, "reset port %d\n", index);
3366 			if (EHCI_PS_IS_LOWSPEED(v) &&
3367 			    (sc->sc_flags & EHCI_SCFLG_TT) == 0) {
3368 				/* Low speed device, give up ownership. */
3369 				DPRINTFN(6, "Low speed device is not support!!!!\n");
3370 				err = USB_ERR_INVAL;
3371 				goto done;
3372 			}
3373 			/* Start reset sequence. */
3374 			v &= ~(EHCI_PS_PE | EHCI_PS_PR);
3375 			EOWRITE4(sc, port, v | EHCI_PS_PR);
3376 
3377 			/* Wait for reset to complete. */
3378 			usb_pause_mtx(&sc->sc_bus.bus_mtx,
3379 			    USB_MS_TO_TICKS(usb_port_root_reset_delay));
3380 
3381 			/* Terminate reset sequence. */
3382 			if (!(sc->sc_flags & EHCI_SCFLG_NORESTERM))
3383 				EOWRITE4(sc, port, v);
3384 
3385 			/* Wait for HC to complete reset. */
3386 			usb_pause_mtx(&sc->sc_bus.bus_mtx,
3387 			    USB_MS_TO_TICKS(EHCI_PORT_RESET_COMPLETE));
3388 
3389 			v = EOREAD4(sc, port);
3390 			DPRINTF("ehci after reset, status=0x%08x\n", v);
3391 			if (v & EHCI_PS_PR) {
3392 				device_printf(sc->sc_bus.bdev,
3393 				    "port reset timeout\n");
3394 				err = USB_ERR_TIMEOUT;
3395 				goto done;
3396 			}
3397 			if (!(v & EHCI_PS_PE) &&
3398 			    (sc->sc_flags & EHCI_SCFLG_TT) == 0) {
3399 				/* Not a high speed device, give up ownership.*/
3400 				DPRINTFN(6, "Full speed device is not support!!!!\n");
3401 				err = USB_ERR_INVAL;
3402 				goto done;
3403 			}
3404 			sc->sc_isreset = 1;
3405 			DPRINTF("ehci port %d reset, status = 0x%08x\n",
3406 			    index, v);
3407 			break;
3408 
3409 		case UHF_PORT_POWER:
3410 			DPRINTFN(3, "set port power %d\n", index);
3411 			EOWRITE4(sc, port, v | EHCI_PS_PP);
3412 			break;
3413 
3414 		case UHF_PORT_TEST:
3415 			DPRINTFN(3, "set port test %d\n", index);
3416 			break;
3417 
3418 		case UHF_PORT_INDICATOR:
3419 			DPRINTFN(3, "set port ind %d\n", index);
3420 			EOWRITE4(sc, port, v | EHCI_PS_PIC);
3421 			break;
3422 
3423 		default:
3424 			err = USB_ERR_IOERROR;
3425 			goto done;
3426 		}
3427 		break;
3428 	case C(UR_CLEAR_TT_BUFFER, UT_WRITE_CLASS_OTHER):
3429 	case C(UR_RESET_TT, UT_WRITE_CLASS_OTHER):
3430 	case C(UR_GET_TT_STATE, UT_READ_CLASS_OTHER):
3431 	case C(UR_STOP_TT, UT_WRITE_CLASS_OTHER):
3432 		break;
3433 	default:
3434 		err = USB_ERR_IOERROR;
3435 		goto done;
3436 	}
3437 done:
3438 	*plength = len;
3439 	*pptr = ptr;
3440 	return (err);
3441 }
3442 
3443 static void
3444 ehci_xfer_setup(struct usb_setup_params *parm)
3445 {
3446 	struct usb_page_search page_info;
3447 	struct usb_page_cache *pc;
3448 	ehci_softc_t *sc;
3449 	struct usb_xfer *xfer;
3450 	void *last_obj;
3451 	uint32_t nqtd;
3452 	uint32_t nqh;
3453 	uint32_t nsitd;
3454 	uint32_t nitd;
3455 	uint32_t n;
3456 
3457 	sc = EHCI_BUS2SC(parm->udev->bus);
3458 	xfer = parm->curr_xfer;
3459 
3460 	nqtd = 0;
3461 	nqh = 0;
3462 	nsitd = 0;
3463 	nitd = 0;
3464 
3465 	/*
3466 	 * compute maximum number of some structures
3467 	 */
3468 	if (parm->methods == &ehci_device_ctrl_methods) {
3469 
3470 		/*
3471 		 * The proof for the "nqtd" formula is illustrated like
3472 		 * this:
3473 		 *
3474 		 * +------------------------------------+
3475 		 * |                                    |
3476 		 * |         |remainder ->              |
3477 		 * |   +-----+---+                      |
3478 		 * |   | xxx | x | frm 0                |
3479 		 * |   +-----+---++                     |
3480 		 * |   | xxx | xx | frm 1               |
3481 		 * |   +-----+----+                     |
3482 		 * |            ...                     |
3483 		 * +------------------------------------+
3484 		 *
3485 		 * "xxx" means a completely full USB transfer descriptor
3486 		 *
3487 		 * "x" and "xx" means a short USB packet
3488 		 *
3489 		 * For the remainder of an USB transfer modulo
3490 		 * "max_data_length" we need two USB transfer descriptors.
3491 		 * One to transfer the remaining data and one to finalise
3492 		 * with a zero length packet in case the "force_short_xfer"
3493 		 * flag is set. We only need two USB transfer descriptors in
3494 		 * the case where the transfer length of the first one is a
3495 		 * factor of "max_frame_size". The rest of the needed USB
3496 		 * transfer descriptors is given by the buffer size divided
3497 		 * by the maximum data payload.
3498 		 */
3499 		parm->hc_max_packet_size = 0x400;
3500 		parm->hc_max_packet_count = 1;
3501 		parm->hc_max_frame_size = EHCI_QTD_PAYLOAD_MAX;
3502 #if USB_HAVE_BUSDMA
3503 		xfer->flags_int.bdma_enable = 1;
3504 #endif
3505 		usbd_transfer_setup_sub(parm);
3506 
3507 		nqh = 1;
3508 		nqtd = ((2 * xfer->nframes) + 1	/* STATUS */
3509 		    + (xfer->max_data_length / xfer->max_hc_frame_size));
3510 
3511 	} else if (parm->methods == &ehci_device_bulk_methods) {
3512 
3513 		parm->hc_max_packet_size = 0x400;
3514 		parm->hc_max_packet_count = 1;
3515 		parm->hc_max_frame_size = EHCI_QTD_PAYLOAD_MAX;
3516 #if USB_HAVE_BUSDMA
3517 		xfer->flags_int.bdma_enable = 1;
3518 #endif
3519 
3520 		usbd_transfer_setup_sub(parm);
3521 
3522 		nqh = 1;
3523 		nqtd = ((2 * xfer->nframes)
3524 		    + (xfer->max_data_length / xfer->max_hc_frame_size));
3525 
3526 	} else if (parm->methods == &ehci_device_intr_methods) {
3527 
3528 		if (parm->speed == USB_SPEED_HIGH) {
3529 			parm->hc_max_packet_size = 0x400;
3530 			parm->hc_max_packet_count = 3;
3531 		} else if (parm->speed == USB_SPEED_FULL) {
3532 			parm->hc_max_packet_size = USB_FS_BYTES_PER_HS_UFRAME;
3533 			parm->hc_max_packet_count = 1;
3534 		} else {
3535 			parm->hc_max_packet_size = USB_FS_BYTES_PER_HS_UFRAME / 8;
3536 			parm->hc_max_packet_count = 1;
3537 		}
3538 
3539 		parm->hc_max_frame_size = EHCI_QTD_PAYLOAD_MAX;
3540 #if USB_HAVE_BUSDMA
3541 		xfer->flags_int.bdma_enable = 1;
3542 #endif
3543 
3544 		usbd_transfer_setup_sub(parm);
3545 
3546 		nqh = 1;
3547 		nqtd = ((2 * xfer->nframes)
3548 		    + (xfer->max_data_length / xfer->max_hc_frame_size));
3549 
3550 	} else if (parm->methods == &ehci_device_isoc_fs_methods) {
3551 
3552 		parm->hc_max_packet_size = 0x3FF;
3553 		parm->hc_max_packet_count = 1;
3554 		parm->hc_max_frame_size = 0x3FF;
3555 #if USB_HAVE_BUSDMA
3556 		xfer->flags_int.bdma_enable = 1;
3557 #endif
3558 
3559 		usbd_transfer_setup_sub(parm);
3560 
3561 		nsitd = xfer->nframes;
3562 
3563 	} else if (parm->methods == &ehci_device_isoc_hs_methods) {
3564 
3565 		parm->hc_max_packet_size = 0x400;
3566 		parm->hc_max_packet_count = 3;
3567 		parm->hc_max_frame_size = 0xC00;
3568 #if USB_HAVE_BUSDMA
3569 		xfer->flags_int.bdma_enable = 1;
3570 #endif
3571 
3572 		usbd_transfer_setup_sub(parm);
3573 
3574 		nitd = ((xfer->nframes + 7) / 8) <<
3575 		    usbd_xfer_get_fps_shift(xfer);
3576 
3577 	} else {
3578 
3579 		parm->hc_max_packet_size = 0x400;
3580 		parm->hc_max_packet_count = 1;
3581 		parm->hc_max_frame_size = 0x400;
3582 
3583 		usbd_transfer_setup_sub(parm);
3584 	}
3585 
3586 alloc_dma_set:
3587 
3588 	if (parm->err) {
3589 		return;
3590 	}
3591 	/*
3592 	 * Allocate queue heads and transfer descriptors
3593 	 */
3594 	last_obj = NULL;
3595 
3596 	if (usbd_transfer_setup_sub_malloc(
3597 	    parm, &pc, sizeof(ehci_itd_t),
3598 	    EHCI_ITD_ALIGN, nitd)) {
3599 		parm->err = USB_ERR_NOMEM;
3600 		return;
3601 	}
3602 	if (parm->buf) {
3603 		for (n = 0; n != nitd; n++) {
3604 			ehci_itd_t *td;
3605 
3606 			usbd_get_page(pc + n, 0, &page_info);
3607 
3608 			td = (ehci_itd_t *)page_info.buffer;
3609 
3610 			/* init TD */
3611 #if USB_HAVE_BUSDMA
3612 			td->itd_self = htohc32(sc, page_info.physaddr | EHCI_LINK_ITD);
3613 #else
3614 			td->itd_self = htohc32(sc, (unsigned int)page_info.buffer | EHCI_LINK_ITD);
3615 #endif
3616 			td->obj_next = (ehci_itd_t *)last_obj;
3617 			td->page_cache = pc + n;
3618 
3619 			last_obj = td;
3620 
3621 			usb_pc_cpu_flush(pc + n);
3622 		}
3623 	}
3624 	if (usbd_transfer_setup_sub_malloc(
3625 	    parm, &pc, sizeof(ehci_sitd_t),
3626 	    EHCI_SITD_ALIGN, nsitd)) {
3627 		parm->err = USB_ERR_NOMEM;
3628 		return;
3629 	}
3630 	if (parm->buf) {
3631 		for (n = 0; n != nsitd; n++) {
3632 			ehci_sitd_t *td;
3633 
3634 			usbd_get_page(pc + n, 0, &page_info);
3635 
3636 			td = (ehci_sitd_t *)page_info.buffer;
3637 
3638 			/* init TD */
3639 #if USB_HAVE_BUSDMA
3640 			td->sitd_self = htohc32(sc, page_info.physaddr | EHCI_LINK_SITD);
3641 #else
3642 			td->sitd_self = htohc32(sc, (unsigned int)page_info.buffer | EHCI_LINK_SITD);
3643 #endif
3644 			td->obj_next = (ehci_sitd_t *)last_obj;
3645 			td->page_cache = pc + n;
3646 
3647 			last_obj = td;
3648 
3649 			usb_pc_cpu_flush(pc + n);
3650 		}
3651 	}
3652 	if (usbd_transfer_setup_sub_malloc(
3653 	    parm, &pc, sizeof(ehci_qtd_t),
3654 	    EHCI_QTD_ALIGN, nqtd)) {
3655 		parm->err = USB_ERR_NOMEM;
3656 		return;
3657 	}
3658 	if (parm->buf) {
3659 		for (n = 0; n != nqtd; n++) {
3660 			ehci_qtd_t *qtd;
3661 
3662 			usbd_get_page(pc + n, 0, &page_info);
3663 
3664 			qtd = (ehci_qtd_t *)page_info.buffer;
3665 
3666 			/* init TD */
3667 #if USB_HAVE_BUSDMA
3668 			qtd->qtd_self = htohc32(sc, page_info.physaddr);
3669 #else
3670 			qtd->qtd_self = htohc32(sc, (unsigned int)page_info.buffer);
3671 #endif
3672 			qtd->obj_next = (ehci_qtd_t *)last_obj;
3673 			qtd->page_cache = pc + n;
3674 
3675 			last_obj = qtd;
3676 
3677 			usb_pc_cpu_flush(pc + n);
3678 		}
3679 	}
3680 	xfer->td_start[xfer->flags_int.curr_dma_set] = last_obj;
3681 
3682 	last_obj = NULL;
3683 
3684 	if (usbd_transfer_setup_sub_malloc(
3685 	    parm, &pc, sizeof(ehci_qh_t),
3686 	    EHCI_QH_ALIGN, nqh)) {
3687 		parm->err = USB_ERR_NOMEM;
3688 		return;
3689 	}
3690 	if (parm->buf) {
3691 		for (n = 0; n != nqh; n++) {
3692 			ehci_qh_t *qh;
3693 
3694 			usbd_get_page(pc + n, 0, &page_info);
3695 
3696 			qh = (ehci_qh_t *)page_info.buffer;
3697 
3698 			/* init QH */
3699 #if USB_HAVE_BUSDMA
3700 			qh->qh_self = htohc32(sc, page_info.physaddr | EHCI_LINK_QH);
3701 #else
3702 			qh->qh_self = htohc32(sc, (unsigned int)page_info.buffer | EHCI_LINK_QH);
3703 #endif
3704 			qh->obj_next = (ehci_qh_t *)last_obj;
3705 			qh->page_cache = pc + n;
3706 
3707 			last_obj = qh;
3708 
3709 			usb_pc_cpu_flush(pc + n);
3710 		}
3711 	}
3712 	xfer->qh_start[xfer->flags_int.curr_dma_set] = last_obj;
3713 
3714 	if (!xfer->flags_int.curr_dma_set) {
3715 		xfer->flags_int.curr_dma_set = 1;
3716 		goto alloc_dma_set;
3717 	}
3718 }
3719 
3720 static void
3721 ehci_xfer_unsetup(struct usb_xfer *xfer)
3722 {
3723 	return;
3724 }
3725 
3726 static void
3727 ehci_ep_init(struct usb_device *udev, struct usb_endpoint_descriptor *edesc,
3728     struct usb_endpoint *ep)
3729 {
3730 	ehci_softc_t *sc = EHCI_BUS2SC(udev->bus);
3731 
3732 	DPRINTFN(2, "endpoint=%p, addr=%d, endpt=%d, mode=%d (%d)\n",
3733 	    ep, udev->address,
3734 	    edesc->bEndpointAddress, udev->flags.usb_mode,
3735 	    sc->sc_addr);
3736 
3737 	if (udev->device_index != sc->sc_addr) {
3738 
3739 		if ((udev->speed != USB_SPEED_HIGH) &&
3740 		    ((udev->hs_hub_addr == 0) ||
3741 		    (udev->hs_port_no == 0) ||
3742 		    (udev->parent_hs_hub == NULL) ||
3743 		    (udev->parent_hs_hub->hub == NULL))) {
3744 			/* We need a transaction translator */
3745 			DPRINTFN(2, "no hub or no port\n");
3746 			goto done;
3747 		}
3748 		switch (edesc->bmAttributes & UE_XFERTYPE) {
3749 		case UE_CONTROL:
3750 			ep->methods = &ehci_device_ctrl_methods;
3751 			DPRINTFN(2, "UE_CONTROL\n");
3752 			break;
3753 		case UE_INTERRUPT:
3754 			ep->methods = &ehci_device_intr_methods;
3755 			DPRINTFN(2, "UE_INTERRUPT\n");
3756 			break;
3757 		case UE_ISOCHRONOUS:
3758 			if (udev->speed == USB_SPEED_HIGH) {
3759 				ep->methods = &ehci_device_isoc_hs_methods;
3760 			} else if (udev->speed == USB_SPEED_FULL) {
3761 				ep->methods = &ehci_device_isoc_fs_methods;
3762 			}
3763 			DPRINTFN(2, "UE_ISOCHRONOUS\n");
3764 			break;
3765 		case UE_BULK:
3766 			ep->methods = &ehci_device_bulk_methods;
3767 			DPRINTFN(2, "UE_BULK\n");
3768 			break;
3769 		default:
3770 			/* do nothing */
3771 			break;
3772 		}
3773 	}
3774 done:
3775 	return;
3776 }
3777 
3778 static void
3779 ehci_get_dma_delay(struct usb_device *udev, uint32_t *pus)
3780 {
3781 	/*
3782 	 * Wait until the hardware has finished any possible use of
3783 	 * the transfer descriptor(s) and QH
3784 	 */
3785 	*pus = (1125);			/* microseconds */
3786 }
3787 
3788 static void
3789 ehci_device_resume(struct usb_device *udev)
3790 {
3791 	ehci_softc_t *sc = EHCI_BUS2SC(udev->bus);
3792 	struct usb_xfer *xfer;
3793 	const struct usb_pipe_methods *methods;
3794 
3795 	DPRINTF("\n");
3796 
3797 	USB_BUS_LOCK(udev->bus);
3798 
3799 	TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
3800 
3801 		if (xfer->xroot->udev == udev) {
3802 
3803 			methods = xfer->endpoint->methods;
3804 
3805 			if ((methods == &ehci_device_bulk_methods) ||
3806 			    (methods == &ehci_device_ctrl_methods)) {
3807 				EHCI_APPEND_QH((ehci_qh_t *)xfer->qh_start[xfer->flags_int.curr_dma_set],
3808 				    sc->sc_async_p_last);
3809 			}
3810 			if (methods == &ehci_device_intr_methods) {
3811 				EHCI_APPEND_QH((ehci_qh_t *)xfer->qh_start[xfer->flags_int.curr_dma_set],
3812 				    sc->sc_intr_p_last[xfer->qh_pos]);
3813 			}
3814 		}
3815 	}
3816 
3817 	USB_BUS_UNLOCK(udev->bus);
3818 
3819 	return;
3820 }
3821 
3822 static void
3823 ehci_device_suspend(struct usb_device *udev)
3824 {
3825 	ehci_softc_t *sc = EHCI_BUS2SC(udev->bus);
3826 	struct usb_xfer *xfer;
3827 	const struct usb_pipe_methods *methods;
3828 
3829 	DPRINTF("\n");
3830 
3831 	USB_BUS_LOCK(udev->bus);
3832 
3833 	TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
3834 
3835 		if (xfer->xroot->udev == udev) {
3836 
3837 			methods = xfer->endpoint->methods;
3838 
3839 			if ((methods == &ehci_device_bulk_methods) ||
3840 			    (methods == &ehci_device_ctrl_methods)) {
3841 				EHCI_REMOVE_QH((ehci_qh_t *)xfer->qh_start[xfer->flags_int.curr_dma_set],
3842 				    sc->sc_async_p_last);
3843 			}
3844 			if (methods == &ehci_device_intr_methods) {
3845 				EHCI_REMOVE_QH((ehci_qh_t *)xfer->qh_start[xfer->flags_int.curr_dma_set],
3846 				    sc->sc_intr_p_last[xfer->qh_pos]);
3847 			}
3848 		}
3849 	}
3850 
3851 	USB_BUS_UNLOCK(udev->bus);
3852 }
3853 
3854 static void
3855 ehci_set_hw_power_sleep(struct usb_bus *bus, uint32_t state)
3856 {
3857 	struct ehci_softc *sc = EHCI_BUS2SC(bus);
3858 
3859 	switch (state) {
3860 	case USB_HW_POWER_SUSPEND:
3861 	case USB_HW_POWER_SHUTDOWN:
3862 		ehci_suspend(sc);
3863 		break;
3864 	case USB_HW_POWER_RESUME:
3865 		ehci_resume(sc);
3866 		break;
3867 	default:
3868 		break;
3869 	}
3870 }
3871 
3872 static void
3873 ehci_set_hw_power(struct usb_bus *bus)
3874 {
3875 	ehci_softc_t *sc = EHCI_BUS2SC(bus);
3876 	uint32_t temp;
3877 	uint32_t flags;
3878 
3879 	DPRINTF("\n");
3880 
3881 	USB_BUS_LOCK(bus);
3882 
3883 	flags = bus->hw_power_state;
3884 
3885 	temp = EOREAD4(sc, EHCI_USBCMD);
3886 
3887 	temp &= ~(EHCI_CMD_ASE | EHCI_CMD_PSE);
3888 
3889 	if (flags & (USB_HW_POWER_CONTROL |
3890 	    USB_HW_POWER_BULK)) {
3891 		DPRINTF("Async is active\n");
3892 		temp |= EHCI_CMD_ASE;
3893 	}
3894 	if (flags & (USB_HW_POWER_INTERRUPT |
3895 	    USB_HW_POWER_ISOC)) {
3896 		DPRINTF("Periodic is active\n");
3897 		temp |= EHCI_CMD_PSE;
3898 	}
3899 	EOWRITE4(sc, EHCI_USBCMD, temp);
3900 
3901 	USB_BUS_UNLOCK(bus);
3902 
3903 	return;
3904 }
3905 
3906 static void
3907 ehci_start_dma_delay_second(struct usb_xfer *xfer)
3908 {
3909 	struct ehci_softc *sc = EHCI_BUS2SC(xfer->xroot->bus);
3910 
3911 	DPRINTF("\n");
3912 
3913 	/* trigger doorbell */
3914 	ehci_doorbell_async(sc);
3915 
3916 	/* give the doorbell 4ms */
3917 	usbd_transfer_timeout_ms(xfer,
3918 	    (void (*)(void *))&usb_dma_delay_done_cb, 4);
3919 }
3920 
3921 /*
3922  * Ring the doorbell twice before freeing any DMA descriptors. Some host
3923  * controllers apparently cache the QH descriptors and need a message
3924  * that the cache needs to be discarded.
3925  */
3926 static void
3927 ehci_start_dma_delay(struct usb_xfer *xfer)
3928 {
3929 	struct ehci_softc *sc = EHCI_BUS2SC(xfer->xroot->bus);
3930 
3931 	DPRINTF("\n");
3932 
3933 	/* trigger doorbell */
3934 	ehci_doorbell_async(sc);
3935 
3936 	/* give the doorbell 4ms */
3937 	usbd_transfer_timeout_ms(xfer,
3938 	    (void (*)(void *))&ehci_start_dma_delay_second, 4);
3939 }
3940 
3941 const struct usb_bus_methods ehci_bus_methods = {
3942 	.endpoint_init = ehci_ep_init,
3943 	.xfer_setup = ehci_xfer_setup,
3944 	.xfer_unsetup = ehci_xfer_unsetup,
3945 	.get_dma_delay = ehci_get_dma_delay,
3946 	.device_resume = ehci_device_resume,
3947 	.device_suspend = ehci_device_suspend,
3948 	.set_hw_power = ehci_set_hw_power,
3949 	.set_hw_power_sleep = ehci_set_hw_power_sleep,
3950 	.roothub_exec = ehci_roothub_exec,
3951 	.xfer_poll = ehci_do_poll,
3952 	.start_dma_delay = ehci_start_dma_delay,
3953 };
3954 #undef USB_DEBUG_VAR
3955