1 /*
2 * Copyright (c) 2001-2004 by David Brownell
3 * Copyright (c) 2003 Michal Sojka, for high-speed iso transfers
4 *
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License as published by the
7 * Free Software Foundation; either version 2 of the License, or (at your
8 * option) any later version.
9 *
10 * This program is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
12 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software Foundation,
17 * Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
18 */
19
20 /* this file is part of ehci-hcd.c */
21
22 /*-------------------------------------------------------------------------*/
23
24 /*
25 * EHCI scheduled transaction support: interrupt, iso, split iso
26 * These are called "periodic" transactions in the EHCI spec.
27 *
28 * Note that for interrupt transfers, the QH/QTD manipulation is shared
29 * with the "asynchronous" transaction support (control/bulk transfers).
30 * The only real difference is in how interrupt transfers are scheduled.
31 *
32 * For ISO, we make an "iso_stream" head to serve the same role as a QH.
33 * It keeps track of every ITD (or SITD) that's linked, and holds enough
34 * pre-calculated schedule data to make appending to the queue be quick.
35 */
36
37 static int ehci_get_frame(struct usb_hcd *hcd);
38
39 /*
40 * periodic_next_shadow - return "next" pointer on shadow list
41 * @periodic: host pointer to qh/itd/sitd
42 * @tag: hardware tag for type of this record
43 */
44 static union ehci_shadow *
periodic_next_shadow(struct ehci_hcd * ehci,union ehci_shadow * periodic,__hc32 tag)45 periodic_next_shadow(struct ehci_hcd *ehci, union ehci_shadow *periodic,
46 __hc32 tag)
47 {
48 switch (hc32_to_cpu(ehci, tag)) {
49 case Q_TYPE_QH:
50 return &periodic->qh->qh_next;
51 case Q_TYPE_FSTN:
52 return &periodic->fstn->fstn_next;
53 case Q_TYPE_ITD:
54 return &periodic->itd->itd_next;
55 /* case Q_TYPE_SITD: */
56 default:
57 return &periodic->sitd->sitd_next;
58 }
59 }
60
61 static __hc32 *
shadow_next_periodic(struct ehci_hcd * ehci,union ehci_shadow * periodic,__hc32 tag)62 shadow_next_periodic(struct ehci_hcd *ehci, union ehci_shadow *periodic,
63 __hc32 tag)
64 {
65 switch (hc32_to_cpu(ehci, tag)) {
66 /* our ehci_shadow.qh is actually software part */
67 case Q_TYPE_QH:
68 return &periodic->qh->hw->hw_next;
69 /* others are hw parts */
70 default:
71 return periodic->hw_next;
72 }
73 }
74
75 /* caller must hold ehci->lock */
periodic_unlink(struct ehci_hcd * ehci,unsigned frame,void * ptr)76 static void periodic_unlink(struct ehci_hcd *ehci, unsigned frame, void *ptr)
77 {
78 union ehci_shadow *prev_p = &ehci->pshadow[frame];
79 __hc32 *hw_p = &ehci->periodic[frame];
80 union ehci_shadow here = *prev_p;
81
82 /* find predecessor of "ptr"; hw and shadow lists are in sync */
83 while (here.ptr && here.ptr != ptr) {
84 prev_p = periodic_next_shadow(ehci, prev_p,
85 Q_NEXT_TYPE(ehci, *hw_p));
86 hw_p = shadow_next_periodic(ehci, &here,
87 Q_NEXT_TYPE(ehci, *hw_p));
88 here = *prev_p;
89 }
90 /* an interrupt entry (at list end) could have been shared */
91 if (!here.ptr)
92 return;
93
94 /* update shadow and hardware lists ... the old "next" pointers
95 * from ptr may still be in use, the caller updates them.
96 */
97 *prev_p = *periodic_next_shadow(ehci, &here,
98 Q_NEXT_TYPE(ehci, *hw_p));
99
100 if (!ehci->use_dummy_qh ||
101 *shadow_next_periodic(ehci, &here, Q_NEXT_TYPE(ehci, *hw_p))
102 != EHCI_LIST_END(ehci))
103 *hw_p = *shadow_next_periodic(ehci, &here,
104 Q_NEXT_TYPE(ehci, *hw_p));
105 else
106 *hw_p = cpu_to_hc32(ehci, ehci->dummy->qh_dma);
107 }
108
109 /*-------------------------------------------------------------------------*/
110
111 /* Bandwidth and TT management */
112
113 /* Find the TT data structure for this device; create it if necessary */
find_tt(struct usb_device * udev)114 static struct ehci_tt *find_tt(struct usb_device *udev)
115 {
116 struct usb_tt *utt = udev->tt;
117 struct ehci_tt *tt, **tt_index, **ptt;
118 unsigned port;
119 bool allocated_index = false;
120
121 if (!utt)
122 return NULL; /* Not below a TT */
123
124 /*
125 * Find/create our data structure.
126 * For hubs with a single TT, we get it directly.
127 * For hubs with multiple TTs, there's an extra level of pointers.
128 */
129 tt_index = NULL;
130 if (utt->multi) {
131 tt_index = utt->hcpriv;
132 if (!tt_index) { /* Create the index array */
133 tt_index = kzalloc(utt->hub->maxchild *
134 sizeof(*tt_index), GFP_ATOMIC);
135 if (!tt_index)
136 return ERR_PTR(-ENOMEM);
137 utt->hcpriv = tt_index;
138 allocated_index = true;
139 }
140 port = udev->ttport - 1;
141 ptt = &tt_index[port];
142 } else {
143 port = 0;
144 ptt = (struct ehci_tt **) &utt->hcpriv;
145 }
146
147 tt = *ptt;
148 if (!tt) { /* Create the ehci_tt */
149 struct ehci_hcd *ehci =
150 hcd_to_ehci(bus_to_hcd(udev->bus));
151
152 tt = kzalloc(sizeof(*tt), GFP_ATOMIC);
153 if (!tt) {
154 if (allocated_index) {
155 utt->hcpriv = NULL;
156 kfree(tt_index);
157 }
158 return ERR_PTR(-ENOMEM);
159 }
160 list_add_tail(&tt->tt_list, &ehci->tt_list);
161 INIT_LIST_HEAD(&tt->ps_list);
162 tt->usb_tt = utt;
163 tt->tt_port = port;
164 *ptt = tt;
165 }
166
167 return tt;
168 }
169
170 /* Release the TT above udev, if it's not in use */
drop_tt(struct usb_device * udev)171 static void drop_tt(struct usb_device *udev)
172 {
173 struct usb_tt *utt = udev->tt;
174 struct ehci_tt *tt, **tt_index, **ptt;
175 int cnt, i;
176
177 if (!utt || !utt->hcpriv)
178 return; /* Not below a TT, or never allocated */
179
180 cnt = 0;
181 if (utt->multi) {
182 tt_index = utt->hcpriv;
183 ptt = &tt_index[udev->ttport - 1];
184
185 /* How many entries are left in tt_index? */
186 for (i = 0; i < utt->hub->maxchild; ++i)
187 cnt += !!tt_index[i];
188 } else {
189 tt_index = NULL;
190 ptt = (struct ehci_tt **) &utt->hcpriv;
191 }
192
193 tt = *ptt;
194 if (!tt || !list_empty(&tt->ps_list))
195 return; /* never allocated, or still in use */
196
197 list_del(&tt->tt_list);
198 *ptt = NULL;
199 kfree(tt);
200 if (cnt == 1) {
201 utt->hcpriv = NULL;
202 kfree(tt_index);
203 }
204 }
205
bandwidth_dbg(struct ehci_hcd * ehci,int sign,char * type,struct ehci_per_sched * ps)206 static void bandwidth_dbg(struct ehci_hcd *ehci, int sign, char *type,
207 struct ehci_per_sched *ps)
208 {
209 dev_dbg(&ps->udev->dev,
210 "ep %02x: %s %s @ %u+%u (%u.%u+%u) [%u/%u us] mask %04x\n",
211 ps->ep->desc.bEndpointAddress,
212 (sign >= 0 ? "reserve" : "release"), type,
213 (ps->bw_phase << 3) + ps->phase_uf, ps->bw_uperiod,
214 ps->phase, ps->phase_uf, ps->period,
215 ps->usecs, ps->c_usecs, ps->cs_mask);
216 }
217
reserve_release_intr_bandwidth(struct ehci_hcd * ehci,struct ehci_qh * qh,int sign)218 static void reserve_release_intr_bandwidth(struct ehci_hcd *ehci,
219 struct ehci_qh *qh, int sign)
220 {
221 unsigned start_uf;
222 unsigned i, j, m;
223 int usecs = qh->ps.usecs;
224 int c_usecs = qh->ps.c_usecs;
225 int tt_usecs = qh->ps.tt_usecs;
226 struct ehci_tt *tt;
227
228 if (qh->ps.phase == NO_FRAME) /* Bandwidth wasn't reserved */
229 return;
230 start_uf = qh->ps.bw_phase << 3;
231
232 bandwidth_dbg(ehci, sign, "intr", &qh->ps);
233
234 if (sign < 0) { /* Release bandwidth */
235 usecs = -usecs;
236 c_usecs = -c_usecs;
237 tt_usecs = -tt_usecs;
238 }
239
240 /* Entire transaction (high speed) or start-split (full/low speed) */
241 for (i = start_uf + qh->ps.phase_uf; i < EHCI_BANDWIDTH_SIZE;
242 i += qh->ps.bw_uperiod)
243 ehci->bandwidth[i] += usecs;
244
245 /* Complete-split (full/low speed) */
246 if (qh->ps.c_usecs) {
247 /* NOTE: adjustments needed for FSTN */
248 for (i = start_uf; i < EHCI_BANDWIDTH_SIZE;
249 i += qh->ps.bw_uperiod) {
250 for ((j = 2, m = 1 << (j+8)); j < 8; (++j, m <<= 1)) {
251 if (qh->ps.cs_mask & m)
252 ehci->bandwidth[i+j] += c_usecs;
253 }
254 }
255 }
256
257 /* FS/LS bus bandwidth */
258 if (tt_usecs) {
259 tt = find_tt(qh->ps.udev);
260 if (sign > 0)
261 list_add_tail(&qh->ps.ps_list, &tt->ps_list);
262 else
263 list_del(&qh->ps.ps_list);
264
265 for (i = start_uf >> 3; i < EHCI_BANDWIDTH_FRAMES;
266 i += qh->ps.bw_period)
267 tt->bandwidth[i] += tt_usecs;
268 }
269 }
270
271 /*-------------------------------------------------------------------------*/
272
compute_tt_budget(u8 budget_table[EHCI_BANDWIDTH_SIZE],struct ehci_tt * tt)273 static void compute_tt_budget(u8 budget_table[EHCI_BANDWIDTH_SIZE],
274 struct ehci_tt *tt)
275 {
276 struct ehci_per_sched *ps;
277 unsigned uframe, uf, x;
278 u8 *budget_line;
279
280 if (!tt)
281 return;
282 memset(budget_table, 0, EHCI_BANDWIDTH_SIZE);
283
284 /* Add up the contributions from all the endpoints using this TT */
285 list_for_each_entry(ps, &tt->ps_list, ps_list) {
286 for (uframe = ps->bw_phase << 3; uframe < EHCI_BANDWIDTH_SIZE;
287 uframe += ps->bw_uperiod) {
288 budget_line = &budget_table[uframe];
289 x = ps->tt_usecs;
290
291 /* propagate the time forward */
292 for (uf = ps->phase_uf; uf < 8; ++uf) {
293 x += budget_line[uf];
294
295 /* Each microframe lasts 125 us */
296 if (x <= 125) {
297 budget_line[uf] = x;
298 break;
299 }
300 budget_line[uf] = 125;
301 x -= 125;
302 }
303 }
304 }
305 }
306
same_tt(struct usb_device * dev1,struct usb_device * dev2)307 static int __maybe_unused same_tt(struct usb_device *dev1,
308 struct usb_device *dev2)
309 {
310 if (!dev1->tt || !dev2->tt)
311 return 0;
312 if (dev1->tt != dev2->tt)
313 return 0;
314 if (dev1->tt->multi)
315 return dev1->ttport == dev2->ttport;
316 else
317 return 1;
318 }
319
320 #ifdef CONFIG_USB_EHCI_TT_NEWSCHED
321
322 /* Which uframe does the low/fullspeed transfer start in?
323 *
324 * The parameter is the mask of ssplits in "H-frame" terms
325 * and this returns the transfer start uframe in "B-frame" terms,
326 * which allows both to match, e.g. a ssplit in "H-frame" uframe 0
327 * will cause a transfer in "B-frame" uframe 0. "B-frames" lag
328 * "H-frames" by 1 uframe. See the EHCI spec sec 4.5 and figure 4.7.
329 */
tt_start_uframe(struct ehci_hcd * ehci,__hc32 mask)330 static inline unsigned char tt_start_uframe(struct ehci_hcd *ehci, __hc32 mask)
331 {
332 unsigned char smask = hc32_to_cpu(ehci, mask) & QH_SMASK;
333
334 if (!smask) {
335 ehci_err(ehci, "invalid empty smask!\n");
336 /* uframe 7 can't have bw so this will indicate failure */
337 return 7;
338 }
339 return ffs(smask) - 1;
340 }
341
342 static const unsigned char
343 max_tt_usecs[] = { 125, 125, 125, 125, 125, 125, 30, 0 };
344
345 /* carryover low/fullspeed bandwidth that crosses uframe boundries */
carryover_tt_bandwidth(unsigned short tt_usecs[8])346 static inline void carryover_tt_bandwidth(unsigned short tt_usecs[8])
347 {
348 int i;
349
350 for (i = 0; i < 7; i++) {
351 if (max_tt_usecs[i] < tt_usecs[i]) {
352 tt_usecs[i+1] += tt_usecs[i] - max_tt_usecs[i];
353 tt_usecs[i] = max_tt_usecs[i];
354 }
355 }
356 }
357
358 /*
359 * Return true if the device's tt's downstream bus is available for a
360 * periodic transfer of the specified length (usecs), starting at the
361 * specified frame/uframe. Note that (as summarized in section 11.19
362 * of the usb 2.0 spec) TTs can buffer multiple transactions for each
363 * uframe.
364 *
365 * The uframe parameter is when the fullspeed/lowspeed transfer
366 * should be executed in "B-frame" terms, which is the same as the
367 * highspeed ssplit's uframe (which is in "H-frame" terms). For example
368 * a ssplit in "H-frame" 0 causes a transfer in "B-frame" 0.
369 * See the EHCI spec sec 4.5 and fig 4.7.
370 *
371 * This checks if the full/lowspeed bus, at the specified starting uframe,
372 * has the specified bandwidth available, according to rules listed
373 * in USB 2.0 spec section 11.18.1 fig 11-60.
374 *
375 * This does not check if the transfer would exceed the max ssplit
376 * limit of 16, specified in USB 2.0 spec section 11.18.4 requirement #4,
377 * since proper scheduling limits ssplits to less than 16 per uframe.
378 */
tt_available(struct ehci_hcd * ehci,struct ehci_per_sched * ps,struct ehci_tt * tt,unsigned frame,unsigned uframe)379 static int tt_available(
380 struct ehci_hcd *ehci,
381 struct ehci_per_sched *ps,
382 struct ehci_tt *tt,
383 unsigned frame,
384 unsigned uframe
385 )
386 {
387 unsigned period = ps->bw_period;
388 unsigned usecs = ps->tt_usecs;
389
390 if ((period == 0) || (uframe >= 7)) /* error */
391 return 0;
392
393 for (frame &= period - 1; frame < EHCI_BANDWIDTH_FRAMES;
394 frame += period) {
395 unsigned i, uf;
396 unsigned short tt_usecs[8];
397
398 if (tt->bandwidth[frame] + usecs > 900)
399 return 0;
400
401 uf = frame << 3;
402 for (i = 0; i < 8; (++i, ++uf))
403 tt_usecs[i] = ehci->tt_budget[uf];
404
405 if (max_tt_usecs[uframe] <= tt_usecs[uframe])
406 return 0;
407
408 /* special case for isoc transfers larger than 125us:
409 * the first and each subsequent fully used uframe
410 * must be empty, so as to not illegally delay
411 * already scheduled transactions
412 */
413 if (usecs > 125) {
414 int ufs = (usecs / 125);
415
416 for (i = uframe; i < (uframe + ufs) && i < 8; i++)
417 if (tt_usecs[i] > 0)
418 return 0;
419 }
420
421 tt_usecs[uframe] += usecs;
422
423 carryover_tt_bandwidth(tt_usecs);
424
425 /* fail if the carryover pushed bw past the last uframe's limit */
426 if (max_tt_usecs[7] < tt_usecs[7])
427 return 0;
428 }
429
430 return 1;
431 }
432
433 #else
434
435 /* return true iff the device's transaction translator is available
436 * for a periodic transfer starting at the specified frame, using
437 * all the uframes in the mask.
438 */
tt_no_collision(struct ehci_hcd * ehci,unsigned period,struct usb_device * dev,unsigned frame,u32 uf_mask)439 static int tt_no_collision(
440 struct ehci_hcd *ehci,
441 unsigned period,
442 struct usb_device *dev,
443 unsigned frame,
444 u32 uf_mask
445 )
446 {
447 if (period == 0) /* error */
448 return 0;
449
450 /* note bandwidth wastage: split never follows csplit
451 * (different dev or endpoint) until the next uframe.
452 * calling convention doesn't make that distinction.
453 */
454 for (; frame < ehci->periodic_size; frame += period) {
455 union ehci_shadow here;
456 __hc32 type;
457 struct ehci_qh_hw *hw;
458
459 here = ehci->pshadow[frame];
460 type = Q_NEXT_TYPE(ehci, ehci->periodic[frame]);
461 while (here.ptr) {
462 switch (hc32_to_cpu(ehci, type)) {
463 case Q_TYPE_ITD:
464 type = Q_NEXT_TYPE(ehci, here.itd->hw_next);
465 here = here.itd->itd_next;
466 continue;
467 case Q_TYPE_QH:
468 hw = here.qh->hw;
469 if (same_tt(dev, here.qh->ps.udev)) {
470 u32 mask;
471
472 mask = hc32_to_cpu(ehci,
473 hw->hw_info2);
474 /* "knows" no gap is needed */
475 mask |= mask >> 8;
476 if (mask & uf_mask)
477 break;
478 }
479 type = Q_NEXT_TYPE(ehci, hw->hw_next);
480 here = here.qh->qh_next;
481 continue;
482 case Q_TYPE_SITD:
483 if (same_tt(dev, here.sitd->urb->dev)) {
484 u16 mask;
485
486 mask = hc32_to_cpu(ehci, here.sitd
487 ->hw_uframe);
488 /* FIXME assumes no gap for IN! */
489 mask |= mask >> 8;
490 if (mask & uf_mask)
491 break;
492 }
493 type = Q_NEXT_TYPE(ehci, here.sitd->hw_next);
494 here = here.sitd->sitd_next;
495 continue;
496 /* case Q_TYPE_FSTN: */
497 default:
498 ehci_dbg(ehci,
499 "periodic frame %d bogus type %d\n",
500 frame, type);
501 }
502
503 /* collision or error */
504 return 0;
505 }
506 }
507
508 /* no collision */
509 return 1;
510 }
511
512 #endif /* CONFIG_USB_EHCI_TT_NEWSCHED */
513
514 /*-------------------------------------------------------------------------*/
515
enable_periodic(struct ehci_hcd * ehci)516 static void enable_periodic(struct ehci_hcd *ehci)
517 {
518 if (ehci->periodic_count++)
519 return;
520
521 /* Stop waiting to turn off the periodic schedule */
522 ehci->enabled_hrtimer_events &= ~BIT(EHCI_HRTIMER_DISABLE_PERIODIC);
523
524 /* Don't start the schedule until PSS is 0 */
525 ehci_poll_PSS(ehci);
526 turn_on_io_watchdog(ehci);
527 }
528
disable_periodic(struct ehci_hcd * ehci)529 static void disable_periodic(struct ehci_hcd *ehci)
530 {
531 if (--ehci->periodic_count)
532 return;
533
534 /* Don't turn off the schedule until PSS is 1 */
535 ehci_poll_PSS(ehci);
536 }
537
538 /*-------------------------------------------------------------------------*/
539
540 /* periodic schedule slots have iso tds (normal or split) first, then a
541 * sparse tree for active interrupt transfers.
542 *
543 * this just links in a qh; caller guarantees uframe masks are set right.
544 * no FSTN support (yet; ehci 0.96+)
545 */
qh_link_periodic(struct ehci_hcd * ehci,struct ehci_qh * qh)546 static void qh_link_periodic(struct ehci_hcd *ehci, struct ehci_qh *qh)
547 {
548 unsigned i;
549 unsigned period = qh->ps.period;
550
551 dev_dbg(&qh->ps.udev->dev,
552 "link qh%d-%04x/%p start %d [%d/%d us]\n",
553 period, hc32_to_cpup(ehci, &qh->hw->hw_info2)
554 & (QH_CMASK | QH_SMASK),
555 qh, qh->ps.phase, qh->ps.usecs, qh->ps.c_usecs);
556
557 /* high bandwidth, or otherwise every microframe */
558 if (period == 0)
559 period = 1;
560
561 for (i = qh->ps.phase; i < ehci->periodic_size; i += period) {
562 union ehci_shadow *prev = &ehci->pshadow[i];
563 __hc32 *hw_p = &ehci->periodic[i];
564 union ehci_shadow here = *prev;
565 __hc32 type = 0;
566
567 /* skip the iso nodes at list head */
568 while (here.ptr) {
569 type = Q_NEXT_TYPE(ehci, *hw_p);
570 if (type == cpu_to_hc32(ehci, Q_TYPE_QH))
571 break;
572 prev = periodic_next_shadow(ehci, prev, type);
573 hw_p = shadow_next_periodic(ehci, &here, type);
574 here = *prev;
575 }
576
577 /* sorting each branch by period (slow-->fast)
578 * enables sharing interior tree nodes
579 */
580 while (here.ptr && qh != here.qh) {
581 if (qh->ps.period > here.qh->ps.period)
582 break;
583 prev = &here.qh->qh_next;
584 hw_p = &here.qh->hw->hw_next;
585 here = *prev;
586 }
587 /* link in this qh, unless some earlier pass did that */
588 if (qh != here.qh) {
589 qh->qh_next = here;
590 if (here.qh)
591 qh->hw->hw_next = *hw_p;
592 wmb();
593 prev->qh = qh;
594 *hw_p = QH_NEXT(ehci, qh->qh_dma);
595 }
596 }
597 qh->qh_state = QH_STATE_LINKED;
598 qh->xacterrs = 0;
599 qh->unlink_reason = 0;
600
601 /* update per-qh bandwidth for debugfs */
602 ehci_to_hcd(ehci)->self.bandwidth_allocated += qh->ps.bw_period
603 ? ((qh->ps.usecs + qh->ps.c_usecs) / qh->ps.bw_period)
604 : (qh->ps.usecs * 8);
605
606 list_add(&qh->intr_node, &ehci->intr_qh_list);
607
608 /* maybe enable periodic schedule processing */
609 ++ehci->intr_count;
610 enable_periodic(ehci);
611 }
612
qh_unlink_periodic(struct ehci_hcd * ehci,struct ehci_qh * qh)613 static void qh_unlink_periodic(struct ehci_hcd *ehci, struct ehci_qh *qh)
614 {
615 unsigned i;
616 unsigned period;
617
618 /*
619 * If qh is for a low/full-speed device, simply unlinking it
620 * could interfere with an ongoing split transaction. To unlink
621 * it safely would require setting the QH_INACTIVATE bit and
622 * waiting at least one frame, as described in EHCI 4.12.2.5.
623 *
624 * We won't bother with any of this. Instead, we assume that the
625 * only reason for unlinking an interrupt QH while the current URB
626 * is still active is to dequeue all the URBs (flush the whole
627 * endpoint queue).
628 *
629 * If rebalancing the periodic schedule is ever implemented, this
630 * approach will no longer be valid.
631 */
632
633 /* high bandwidth, or otherwise part of every microframe */
634 period = qh->ps.period ? : 1;
635
636 for (i = qh->ps.phase; i < ehci->periodic_size; i += period)
637 periodic_unlink(ehci, i, qh);
638
639 /* update per-qh bandwidth for debugfs */
640 ehci_to_hcd(ehci)->self.bandwidth_allocated -= qh->ps.bw_period
641 ? ((qh->ps.usecs + qh->ps.c_usecs) / qh->ps.bw_period)
642 : (qh->ps.usecs * 8);
643
644 dev_dbg(&qh->ps.udev->dev,
645 "unlink qh%d-%04x/%p start %d [%d/%d us]\n",
646 qh->ps.period,
647 hc32_to_cpup(ehci, &qh->hw->hw_info2) & (QH_CMASK | QH_SMASK),
648 qh, qh->ps.phase, qh->ps.usecs, qh->ps.c_usecs);
649
650 /* qh->qh_next still "live" to HC */
651 qh->qh_state = QH_STATE_UNLINK;
652 qh->qh_next.ptr = NULL;
653
654 if (ehci->qh_scan_next == qh)
655 ehci->qh_scan_next = list_entry(qh->intr_node.next,
656 struct ehci_qh, intr_node);
657 list_del(&qh->intr_node);
658 }
659
cancel_unlink_wait_intr(struct ehci_hcd * ehci,struct ehci_qh * qh)660 static void cancel_unlink_wait_intr(struct ehci_hcd *ehci, struct ehci_qh *qh)
661 {
662 if (qh->qh_state != QH_STATE_LINKED ||
663 list_empty(&qh->unlink_node))
664 return;
665
666 list_del_init(&qh->unlink_node);
667
668 /*
669 * TODO: disable the event of EHCI_HRTIMER_START_UNLINK_INTR for
670 * avoiding unnecessary CPU wakeup
671 */
672 }
673
start_unlink_intr(struct ehci_hcd * ehci,struct ehci_qh * qh)674 static void start_unlink_intr(struct ehci_hcd *ehci, struct ehci_qh *qh)
675 {
676 /* If the QH isn't linked then there's nothing we can do. */
677 if (qh->qh_state != QH_STATE_LINKED)
678 return;
679
680 /* if the qh is waiting for unlink, cancel it now */
681 cancel_unlink_wait_intr(ehci, qh);
682
683 qh_unlink_periodic(ehci, qh);
684
685 /* Make sure the unlinks are visible before starting the timer */
686 wmb();
687
688 /*
689 * The EHCI spec doesn't say how long it takes the controller to
690 * stop accessing an unlinked interrupt QH. The timer delay is
691 * 9 uframes; presumably that will be long enough.
692 */
693 qh->unlink_cycle = ehci->intr_unlink_cycle;
694
695 /* New entries go at the end of the intr_unlink list */
696 list_add_tail(&qh->unlink_node, &ehci->intr_unlink);
697
698 if (ehci->intr_unlinking)
699 ; /* Avoid recursive calls */
700 else if (ehci->rh_state < EHCI_RH_RUNNING)
701 ehci_handle_intr_unlinks(ehci);
702 else if (ehci->intr_unlink.next == &qh->unlink_node) {
703 ehci_enable_event(ehci, EHCI_HRTIMER_UNLINK_INTR, true);
704 ++ehci->intr_unlink_cycle;
705 }
706 }
707
708 /*
709 * It is common only one intr URB is scheduled on one qh, and
710 * given complete() is run in tasklet context, introduce a bit
711 * delay to avoid unlink qh too early.
712 */
start_unlink_intr_wait(struct ehci_hcd * ehci,struct ehci_qh * qh)713 static void start_unlink_intr_wait(struct ehci_hcd *ehci,
714 struct ehci_qh *qh)
715 {
716 qh->unlink_cycle = ehci->intr_unlink_wait_cycle;
717
718 /* New entries go at the end of the intr_unlink_wait list */
719 list_add_tail(&qh->unlink_node, &ehci->intr_unlink_wait);
720
721 if (ehci->rh_state < EHCI_RH_RUNNING)
722 ehci_handle_start_intr_unlinks(ehci);
723 else if (ehci->intr_unlink_wait.next == &qh->unlink_node) {
724 ehci_enable_event(ehci, EHCI_HRTIMER_START_UNLINK_INTR, true);
725 ++ehci->intr_unlink_wait_cycle;
726 }
727 }
728
end_unlink_intr(struct ehci_hcd * ehci,struct ehci_qh * qh)729 static void end_unlink_intr(struct ehci_hcd *ehci, struct ehci_qh *qh)
730 {
731 struct ehci_qh_hw *hw = qh->hw;
732 int rc;
733
734 qh->qh_state = QH_STATE_IDLE;
735 hw->hw_next = EHCI_LIST_END(ehci);
736
737 if (!list_empty(&qh->qtd_list))
738 qh_completions(ehci, qh);
739
740 /* reschedule QH iff another request is queued */
741 if (!list_empty(&qh->qtd_list) && ehci->rh_state == EHCI_RH_RUNNING) {
742 rc = qh_schedule(ehci, qh);
743 if (rc == 0) {
744 qh_refresh(ehci, qh);
745 qh_link_periodic(ehci, qh);
746 }
747
748 /* An error here likely indicates handshake failure
749 * or no space left in the schedule. Neither fault
750 * should happen often ...
751 *
752 * FIXME kill the now-dysfunctional queued urbs
753 */
754 else {
755 ehci_err(ehci, "can't reschedule qh %p, err %d\n",
756 qh, rc);
757 }
758 }
759
760 /* maybe turn off periodic schedule */
761 --ehci->intr_count;
762 disable_periodic(ehci);
763 }
764
765 /*-------------------------------------------------------------------------*/
766
check_period(struct ehci_hcd * ehci,unsigned frame,unsigned uframe,unsigned uperiod,unsigned usecs)767 static int check_period(
768 struct ehci_hcd *ehci,
769 unsigned frame,
770 unsigned uframe,
771 unsigned uperiod,
772 unsigned usecs
773 ) {
774 /* complete split running into next frame?
775 * given FSTN support, we could sometimes check...
776 */
777 if (uframe >= 8)
778 return 0;
779
780 /* convert "usecs we need" to "max already claimed" */
781 usecs = ehci->uframe_periodic_max - usecs;
782
783 for (uframe += frame << 3; uframe < EHCI_BANDWIDTH_SIZE;
784 uframe += uperiod) {
785 if (ehci->bandwidth[uframe] > usecs)
786 return 0;
787 }
788
789 /* success! */
790 return 1;
791 }
792
check_intr_schedule(struct ehci_hcd * ehci,unsigned frame,unsigned uframe,struct ehci_qh * qh,unsigned * c_maskp,struct ehci_tt * tt)793 static int check_intr_schedule(
794 struct ehci_hcd *ehci,
795 unsigned frame,
796 unsigned uframe,
797 struct ehci_qh *qh,
798 unsigned *c_maskp,
799 struct ehci_tt *tt
800 )
801 {
802 int retval = -ENOSPC;
803 u8 mask = 0;
804
805 if (qh->ps.c_usecs && uframe >= 6) /* FSTN territory? */
806 goto done;
807
808 if (!check_period(ehci, frame, uframe, qh->ps.bw_uperiod, qh->ps.usecs))
809 goto done;
810 if (!qh->ps.c_usecs) {
811 retval = 0;
812 *c_maskp = 0;
813 goto done;
814 }
815
816 #ifdef CONFIG_USB_EHCI_TT_NEWSCHED
817 if (tt_available(ehci, &qh->ps, tt, frame, uframe)) {
818 unsigned i;
819
820 /* TODO : this may need FSTN for SSPLIT in uframe 5. */
821 for (i = uframe+2; i < 8 && i <= uframe+4; i++)
822 if (!check_period(ehci, frame, i,
823 qh->ps.bw_uperiod, qh->ps.c_usecs))
824 goto done;
825 else
826 mask |= 1 << i;
827
828 retval = 0;
829
830 *c_maskp = mask;
831 }
832 #else
833 /* Make sure this tt's buffer is also available for CSPLITs.
834 * We pessimize a bit; probably the typical full speed case
835 * doesn't need the second CSPLIT.
836 *
837 * NOTE: both SPLIT and CSPLIT could be checked in just
838 * one smart pass...
839 */
840 mask = 0x03 << (uframe + qh->gap_uf);
841 *c_maskp = mask;
842
843 mask |= 1 << uframe;
844 if (tt_no_collision(ehci, qh->ps.bw_period, qh->ps.udev, frame, mask)) {
845 if (!check_period(ehci, frame, uframe + qh->gap_uf + 1,
846 qh->ps.bw_uperiod, qh->ps.c_usecs))
847 goto done;
848 if (!check_period(ehci, frame, uframe + qh->gap_uf,
849 qh->ps.bw_uperiod, qh->ps.c_usecs))
850 goto done;
851 retval = 0;
852 }
853 #endif
854 done:
855 return retval;
856 }
857
858 /* "first fit" scheduling policy used the first time through,
859 * or when the previous schedule slot can't be re-used.
860 */
qh_schedule(struct ehci_hcd * ehci,struct ehci_qh * qh)861 static int qh_schedule(struct ehci_hcd *ehci, struct ehci_qh *qh)
862 {
863 int status = 0;
864 unsigned uframe;
865 unsigned c_mask;
866 struct ehci_qh_hw *hw = qh->hw;
867 struct ehci_tt *tt;
868
869 hw->hw_next = EHCI_LIST_END(ehci);
870
871 /* reuse the previous schedule slots, if we can */
872 if (qh->ps.phase != NO_FRAME) {
873 ehci_dbg(ehci, "reused qh %p schedule\n", qh);
874 return 0;
875 }
876
877 uframe = 0;
878 c_mask = 0;
879 tt = find_tt(qh->ps.udev);
880 if (IS_ERR(tt)) {
881 status = PTR_ERR(tt);
882 goto done;
883 }
884 compute_tt_budget(ehci->tt_budget, tt);
885
886 /* else scan the schedule to find a group of slots such that all
887 * uframes have enough periodic bandwidth available.
888 */
889 /* "normal" case, uframing flexible except with splits */
890 if (qh->ps.bw_period) {
891 int i;
892 unsigned frame;
893
894 for (i = qh->ps.bw_period; i > 0; --i) {
895 frame = ++ehci->random_frame & (qh->ps.bw_period - 1);
896 for (uframe = 0; uframe < 8; uframe++) {
897 status = check_intr_schedule(ehci,
898 frame, uframe, qh, &c_mask, tt);
899 if (status == 0)
900 goto got_it;
901 }
902 }
903
904 /* qh->ps.bw_period == 0 means every uframe */
905 } else {
906 status = check_intr_schedule(ehci, 0, 0, qh, &c_mask, tt);
907 }
908 if (status)
909 goto done;
910
911 got_it:
912 qh->ps.phase = (qh->ps.period ? ehci->random_frame &
913 (qh->ps.period - 1) : 0);
914 qh->ps.bw_phase = qh->ps.phase & (qh->ps.bw_period - 1);
915 qh->ps.phase_uf = uframe;
916 qh->ps.cs_mask = qh->ps.period ?
917 (c_mask << 8) | (1 << uframe) :
918 QH_SMASK;
919
920 /* reset S-frame and (maybe) C-frame masks */
921 hw->hw_info2 &= cpu_to_hc32(ehci, ~(QH_CMASK | QH_SMASK));
922 hw->hw_info2 |= cpu_to_hc32(ehci, qh->ps.cs_mask);
923 reserve_release_intr_bandwidth(ehci, qh, 1);
924
925 done:
926 return status;
927 }
928
intr_submit(struct ehci_hcd * ehci,struct urb * urb,struct list_head * qtd_list,gfp_t mem_flags)929 static int intr_submit(
930 struct ehci_hcd *ehci,
931 struct urb *urb,
932 struct list_head *qtd_list,
933 gfp_t mem_flags
934 ) {
935 unsigned epnum;
936 unsigned long flags;
937 struct ehci_qh *qh;
938 int status;
939 struct list_head empty;
940
941 /* get endpoint and transfer/schedule data */
942 epnum = urb->ep->desc.bEndpointAddress;
943
944 spin_lock_irqsave(&ehci->lock, flags);
945
946 if (unlikely(!HCD_HW_ACCESSIBLE(ehci_to_hcd(ehci)))) {
947 status = -ESHUTDOWN;
948 goto done_not_linked;
949 }
950 status = usb_hcd_link_urb_to_ep(ehci_to_hcd(ehci), urb);
951 if (unlikely(status))
952 goto done_not_linked;
953
954 /* get qh and force any scheduling errors */
955 INIT_LIST_HEAD(&empty);
956 qh = qh_append_tds(ehci, urb, &empty, epnum, &urb->ep->hcpriv);
957 if (qh == NULL) {
958 status = -ENOMEM;
959 goto done;
960 }
961 if (qh->qh_state == QH_STATE_IDLE) {
962 status = qh_schedule(ehci, qh);
963 if (status)
964 goto done;
965 }
966
967 /* then queue the urb's tds to the qh */
968 qh = qh_append_tds(ehci, urb, qtd_list, epnum, &urb->ep->hcpriv);
969 BUG_ON(qh == NULL);
970
971 /* stuff into the periodic schedule */
972 if (qh->qh_state == QH_STATE_IDLE) {
973 qh_refresh(ehci, qh);
974 qh_link_periodic(ehci, qh);
975 } else {
976 /* cancel unlink wait for the qh */
977 cancel_unlink_wait_intr(ehci, qh);
978 }
979
980 /* ... update usbfs periodic stats */
981 ehci_to_hcd(ehci)->self.bandwidth_int_reqs++;
982
983 done:
984 if (unlikely(status))
985 usb_hcd_unlink_urb_from_ep(ehci_to_hcd(ehci), urb);
986 done_not_linked:
987 spin_unlock_irqrestore(&ehci->lock, flags);
988 if (status)
989 qtd_list_free(ehci, urb, qtd_list);
990
991 return status;
992 }
993
scan_intr(struct ehci_hcd * ehci)994 static void scan_intr(struct ehci_hcd *ehci)
995 {
996 struct ehci_qh *qh;
997
998 list_for_each_entry_safe(qh, ehci->qh_scan_next, &ehci->intr_qh_list,
999 intr_node) {
1000
1001 /* clean any finished work for this qh */
1002 if (!list_empty(&qh->qtd_list)) {
1003 int temp;
1004
1005 /*
1006 * Unlinks could happen here; completion reporting
1007 * drops the lock. That's why ehci->qh_scan_next
1008 * always holds the next qh to scan; if the next qh
1009 * gets unlinked then ehci->qh_scan_next is adjusted
1010 * in qh_unlink_periodic().
1011 */
1012 temp = qh_completions(ehci, qh);
1013 if (unlikely(temp))
1014 start_unlink_intr(ehci, qh);
1015 else if (unlikely(list_empty(&qh->qtd_list) &&
1016 qh->qh_state == QH_STATE_LINKED))
1017 start_unlink_intr_wait(ehci, qh);
1018 }
1019 }
1020 }
1021
1022 /*-------------------------------------------------------------------------*/
1023
1024 /* ehci_iso_stream ops work with both ITD and SITD */
1025
1026 static struct ehci_iso_stream *
iso_stream_alloc(gfp_t mem_flags)1027 iso_stream_alloc(gfp_t mem_flags)
1028 {
1029 struct ehci_iso_stream *stream;
1030
1031 stream = kzalloc(sizeof(*stream), mem_flags);
1032 if (likely(stream != NULL)) {
1033 INIT_LIST_HEAD(&stream->td_list);
1034 INIT_LIST_HEAD(&stream->free_list);
1035 stream->next_uframe = NO_FRAME;
1036 stream->ps.phase = NO_FRAME;
1037 }
1038 return stream;
1039 }
1040
1041 static void
iso_stream_init(struct ehci_hcd * ehci,struct ehci_iso_stream * stream,struct urb * urb)1042 iso_stream_init(
1043 struct ehci_hcd *ehci,
1044 struct ehci_iso_stream *stream,
1045 struct urb *urb
1046 )
1047 {
1048 static const u8 smask_out[] = { 0x01, 0x03, 0x07, 0x0f, 0x1f, 0x3f };
1049
1050 struct usb_device *dev = urb->dev;
1051 u32 buf1;
1052 unsigned epnum, maxp;
1053 int is_input;
1054 unsigned tmp;
1055
1056 /*
1057 * this might be a "high bandwidth" highspeed endpoint,
1058 * as encoded in the ep descriptor's wMaxPacket field
1059 */
1060 epnum = usb_pipeendpoint(urb->pipe);
1061 is_input = usb_pipein(urb->pipe) ? USB_DIR_IN : 0;
1062 maxp = usb_endpoint_maxp(&urb->ep->desc);
1063 buf1 = is_input ? 1 << 11 : 0;
1064
1065 /* knows about ITD vs SITD */
1066 if (dev->speed == USB_SPEED_HIGH) {
1067 unsigned multi = usb_endpoint_maxp_mult(&urb->ep->desc);
1068
1069 stream->highspeed = 1;
1070
1071 buf1 |= maxp;
1072 maxp *= multi;
1073
1074 stream->buf0 = cpu_to_hc32(ehci, (epnum << 8) | dev->devnum);
1075 stream->buf1 = cpu_to_hc32(ehci, buf1);
1076 stream->buf2 = cpu_to_hc32(ehci, multi);
1077
1078 /* usbfs wants to report the average usecs per frame tied up
1079 * when transfers on this endpoint are scheduled ...
1080 */
1081 stream->ps.usecs = HS_USECS_ISO(maxp);
1082
1083 /* period for bandwidth allocation */
1084 tmp = min_t(unsigned, EHCI_BANDWIDTH_SIZE,
1085 1 << (urb->ep->desc.bInterval - 1));
1086
1087 /* Allow urb->interval to override */
1088 stream->ps.bw_uperiod = min_t(unsigned, tmp, urb->interval);
1089
1090 stream->uperiod = urb->interval;
1091 stream->ps.period = urb->interval >> 3;
1092 stream->bandwidth = stream->ps.usecs * 8 /
1093 stream->ps.bw_uperiod;
1094
1095 } else {
1096 u32 addr;
1097 int think_time;
1098 int hs_transfers;
1099
1100 addr = dev->ttport << 24;
1101 if (!ehci_is_TDI(ehci)
1102 || (dev->tt->hub !=
1103 ehci_to_hcd(ehci)->self.root_hub))
1104 addr |= dev->tt->hub->devnum << 16;
1105 addr |= epnum << 8;
1106 addr |= dev->devnum;
1107 stream->ps.usecs = HS_USECS_ISO(maxp);
1108 think_time = dev->tt->think_time;
1109 stream->ps.tt_usecs = NS_TO_US(think_time + usb_calc_bus_time(
1110 dev->speed, is_input, 1, maxp));
1111 hs_transfers = max(1u, (maxp + 187) / 188);
1112 if (is_input) {
1113 u32 tmp;
1114
1115 addr |= 1 << 31;
1116 stream->ps.c_usecs = stream->ps.usecs;
1117 stream->ps.usecs = HS_USECS_ISO(1);
1118 stream->ps.cs_mask = 1;
1119
1120 /* c-mask as specified in USB 2.0 11.18.4 3.c */
1121 tmp = (1 << (hs_transfers + 2)) - 1;
1122 stream->ps.cs_mask |= tmp << (8 + 2);
1123 } else
1124 stream->ps.cs_mask = smask_out[hs_transfers - 1];
1125
1126 /* period for bandwidth allocation */
1127 tmp = min_t(unsigned, EHCI_BANDWIDTH_FRAMES,
1128 1 << (urb->ep->desc.bInterval - 1));
1129
1130 /* Allow urb->interval to override */
1131 stream->ps.bw_period = min_t(unsigned, tmp, urb->interval);
1132 stream->ps.bw_uperiod = stream->ps.bw_period << 3;
1133
1134 stream->ps.period = urb->interval;
1135 stream->uperiod = urb->interval << 3;
1136 stream->bandwidth = (stream->ps.usecs + stream->ps.c_usecs) /
1137 stream->ps.bw_period;
1138
1139 /* stream->splits gets created from cs_mask later */
1140 stream->address = cpu_to_hc32(ehci, addr);
1141 }
1142
1143 stream->ps.udev = dev;
1144 stream->ps.ep = urb->ep;
1145
1146 stream->bEndpointAddress = is_input | epnum;
1147 stream->maxp = maxp;
1148 }
1149
1150 static struct ehci_iso_stream *
iso_stream_find(struct ehci_hcd * ehci,struct urb * urb)1151 iso_stream_find(struct ehci_hcd *ehci, struct urb *urb)
1152 {
1153 unsigned epnum;
1154 struct ehci_iso_stream *stream;
1155 struct usb_host_endpoint *ep;
1156 unsigned long flags;
1157
1158 epnum = usb_pipeendpoint (urb->pipe);
1159 if (usb_pipein(urb->pipe))
1160 ep = urb->dev->ep_in[epnum];
1161 else
1162 ep = urb->dev->ep_out[epnum];
1163
1164 spin_lock_irqsave(&ehci->lock, flags);
1165 stream = ep->hcpriv;
1166
1167 if (unlikely(stream == NULL)) {
1168 stream = iso_stream_alloc(GFP_ATOMIC);
1169 if (likely(stream != NULL)) {
1170 ep->hcpriv = stream;
1171 iso_stream_init(ehci, stream, urb);
1172 }
1173
1174 /* if dev->ep [epnum] is a QH, hw is set */
1175 } else if (unlikely(stream->hw != NULL)) {
1176 ehci_dbg(ehci, "dev %s ep%d%s, not iso??\n",
1177 urb->dev->devpath, epnum,
1178 usb_pipein(urb->pipe) ? "in" : "out");
1179 stream = NULL;
1180 }
1181
1182 spin_unlock_irqrestore(&ehci->lock, flags);
1183 return stream;
1184 }
1185
1186 /*-------------------------------------------------------------------------*/
1187
1188 /* ehci_iso_sched ops can be ITD-only or SITD-only */
1189
1190 static struct ehci_iso_sched *
iso_sched_alloc(unsigned packets,gfp_t mem_flags)1191 iso_sched_alloc(unsigned packets, gfp_t mem_flags)
1192 {
1193 struct ehci_iso_sched *iso_sched;
1194 int size = sizeof(*iso_sched);
1195
1196 size += packets * sizeof(struct ehci_iso_packet);
1197 iso_sched = kzalloc(size, mem_flags);
1198 if (likely(iso_sched != NULL))
1199 INIT_LIST_HEAD(&iso_sched->td_list);
1200
1201 return iso_sched;
1202 }
1203
1204 static inline void
itd_sched_init(struct ehci_hcd * ehci,struct ehci_iso_sched * iso_sched,struct ehci_iso_stream * stream,struct urb * urb)1205 itd_sched_init(
1206 struct ehci_hcd *ehci,
1207 struct ehci_iso_sched *iso_sched,
1208 struct ehci_iso_stream *stream,
1209 struct urb *urb
1210 )
1211 {
1212 unsigned i;
1213 dma_addr_t dma = urb->transfer_dma;
1214
1215 /* how many uframes are needed for these transfers */
1216 iso_sched->span = urb->number_of_packets * stream->uperiod;
1217
1218 /* figure out per-uframe itd fields that we'll need later
1219 * when we fit new itds into the schedule.
1220 */
1221 for (i = 0; i < urb->number_of_packets; i++) {
1222 struct ehci_iso_packet *uframe = &iso_sched->packet[i];
1223 unsigned length;
1224 dma_addr_t buf;
1225 u32 trans;
1226
1227 length = urb->iso_frame_desc[i].length;
1228 buf = dma + urb->iso_frame_desc[i].offset;
1229
1230 trans = EHCI_ISOC_ACTIVE;
1231 trans |= buf & 0x0fff;
1232 if (unlikely(((i + 1) == urb->number_of_packets))
1233 && !(urb->transfer_flags & URB_NO_INTERRUPT))
1234 trans |= EHCI_ITD_IOC;
1235 trans |= length << 16;
1236 uframe->transaction = cpu_to_hc32(ehci, trans);
1237
1238 /* might need to cross a buffer page within a uframe */
1239 uframe->bufp = (buf & ~(u64)0x0fff);
1240 buf += length;
1241 if (unlikely((uframe->bufp != (buf & ~(u64)0x0fff))))
1242 uframe->cross = 1;
1243 }
1244 }
1245
1246 static void
iso_sched_free(struct ehci_iso_stream * stream,struct ehci_iso_sched * iso_sched)1247 iso_sched_free(
1248 struct ehci_iso_stream *stream,
1249 struct ehci_iso_sched *iso_sched
1250 )
1251 {
1252 if (!iso_sched)
1253 return;
1254 /* caller must hold ehci->lock! */
1255 list_splice(&iso_sched->td_list, &stream->free_list);
1256 kfree(iso_sched);
1257 }
1258
1259 static int
itd_urb_transaction(struct ehci_iso_stream * stream,struct ehci_hcd * ehci,struct urb * urb,gfp_t mem_flags)1260 itd_urb_transaction(
1261 struct ehci_iso_stream *stream,
1262 struct ehci_hcd *ehci,
1263 struct urb *urb,
1264 gfp_t mem_flags
1265 )
1266 {
1267 struct ehci_itd *itd;
1268 dma_addr_t itd_dma;
1269 int i;
1270 unsigned num_itds;
1271 struct ehci_iso_sched *sched;
1272 unsigned long flags;
1273
1274 sched = iso_sched_alloc(urb->number_of_packets, mem_flags);
1275 if (unlikely(sched == NULL))
1276 return -ENOMEM;
1277
1278 itd_sched_init(ehci, sched, stream, urb);
1279
1280 if (urb->interval < 8)
1281 num_itds = 1 + (sched->span + 7) / 8;
1282 else
1283 num_itds = urb->number_of_packets;
1284
1285 /* allocate/init ITDs */
1286 spin_lock_irqsave(&ehci->lock, flags);
1287 for (i = 0; i < num_itds; i++) {
1288
1289 /*
1290 * Use iTDs from the free list, but not iTDs that may
1291 * still be in use by the hardware.
1292 */
1293 if (likely(!list_empty(&stream->free_list))) {
1294 itd = list_first_entry(&stream->free_list,
1295 struct ehci_itd, itd_list);
1296 if (itd->frame == ehci->now_frame)
1297 goto alloc_itd;
1298 list_del(&itd->itd_list);
1299 itd_dma = itd->itd_dma;
1300 } else {
1301 alloc_itd:
1302 spin_unlock_irqrestore(&ehci->lock, flags);
1303 itd = dma_pool_alloc(ehci->itd_pool, mem_flags,
1304 &itd_dma);
1305 spin_lock_irqsave(&ehci->lock, flags);
1306 if (!itd) {
1307 iso_sched_free(stream, sched);
1308 spin_unlock_irqrestore(&ehci->lock, flags);
1309 return -ENOMEM;
1310 }
1311 }
1312
1313 memset(itd, 0, sizeof(*itd));
1314 itd->itd_dma = itd_dma;
1315 itd->frame = NO_FRAME;
1316 list_add(&itd->itd_list, &sched->td_list);
1317 }
1318 spin_unlock_irqrestore(&ehci->lock, flags);
1319
1320 /* temporarily store schedule info in hcpriv */
1321 urb->hcpriv = sched;
1322 urb->error_count = 0;
1323 return 0;
1324 }
1325
1326 /*-------------------------------------------------------------------------*/
1327
reserve_release_iso_bandwidth(struct ehci_hcd * ehci,struct ehci_iso_stream * stream,int sign)1328 static void reserve_release_iso_bandwidth(struct ehci_hcd *ehci,
1329 struct ehci_iso_stream *stream, int sign)
1330 {
1331 unsigned uframe;
1332 unsigned i, j;
1333 unsigned s_mask, c_mask, m;
1334 int usecs = stream->ps.usecs;
1335 int c_usecs = stream->ps.c_usecs;
1336 int tt_usecs = stream->ps.tt_usecs;
1337 struct ehci_tt *tt;
1338
1339 if (stream->ps.phase == NO_FRAME) /* Bandwidth wasn't reserved */
1340 return;
1341 uframe = stream->ps.bw_phase << 3;
1342
1343 bandwidth_dbg(ehci, sign, "iso", &stream->ps);
1344
1345 if (sign < 0) { /* Release bandwidth */
1346 usecs = -usecs;
1347 c_usecs = -c_usecs;
1348 tt_usecs = -tt_usecs;
1349 }
1350
1351 if (!stream->splits) { /* High speed */
1352 for (i = uframe + stream->ps.phase_uf; i < EHCI_BANDWIDTH_SIZE;
1353 i += stream->ps.bw_uperiod)
1354 ehci->bandwidth[i] += usecs;
1355
1356 } else { /* Full speed */
1357 s_mask = stream->ps.cs_mask;
1358 c_mask = s_mask >> 8;
1359
1360 /* NOTE: adjustment needed for frame overflow */
1361 for (i = uframe; i < EHCI_BANDWIDTH_SIZE;
1362 i += stream->ps.bw_uperiod) {
1363 for ((j = stream->ps.phase_uf, m = 1 << j); j < 8;
1364 (++j, m <<= 1)) {
1365 if (s_mask & m)
1366 ehci->bandwidth[i+j] += usecs;
1367 else if (c_mask & m)
1368 ehci->bandwidth[i+j] += c_usecs;
1369 }
1370 }
1371
1372 tt = find_tt(stream->ps.udev);
1373 if (sign > 0)
1374 list_add_tail(&stream->ps.ps_list, &tt->ps_list);
1375 else
1376 list_del(&stream->ps.ps_list);
1377
1378 for (i = uframe >> 3; i < EHCI_BANDWIDTH_FRAMES;
1379 i += stream->ps.bw_period)
1380 tt->bandwidth[i] += tt_usecs;
1381 }
1382 }
1383
1384 static inline int
itd_slot_ok(struct ehci_hcd * ehci,struct ehci_iso_stream * stream,unsigned uframe)1385 itd_slot_ok(
1386 struct ehci_hcd *ehci,
1387 struct ehci_iso_stream *stream,
1388 unsigned uframe
1389 )
1390 {
1391 unsigned usecs;
1392
1393 /* convert "usecs we need" to "max already claimed" */
1394 usecs = ehci->uframe_periodic_max - stream->ps.usecs;
1395
1396 for (uframe &= stream->ps.bw_uperiod - 1; uframe < EHCI_BANDWIDTH_SIZE;
1397 uframe += stream->ps.bw_uperiod) {
1398 if (ehci->bandwidth[uframe] > usecs)
1399 return 0;
1400 }
1401 return 1;
1402 }
1403
1404 static inline int
sitd_slot_ok(struct ehci_hcd * ehci,struct ehci_iso_stream * stream,unsigned uframe,struct ehci_iso_sched * sched,struct ehci_tt * tt)1405 sitd_slot_ok(
1406 struct ehci_hcd *ehci,
1407 struct ehci_iso_stream *stream,
1408 unsigned uframe,
1409 struct ehci_iso_sched *sched,
1410 struct ehci_tt *tt
1411 )
1412 {
1413 unsigned mask, tmp;
1414 unsigned frame, uf;
1415
1416 mask = stream->ps.cs_mask << (uframe & 7);
1417
1418 /* for OUT, don't wrap SSPLIT into H-microframe 7 */
1419 if (((stream->ps.cs_mask & 0xff) << (uframe & 7)) >= (1 << 7))
1420 return 0;
1421
1422 /* for IN, don't wrap CSPLIT into the next frame */
1423 if (mask & ~0xffff)
1424 return 0;
1425
1426 /* check bandwidth */
1427 uframe &= stream->ps.bw_uperiod - 1;
1428 frame = uframe >> 3;
1429
1430 #ifdef CONFIG_USB_EHCI_TT_NEWSCHED
1431 /* The tt's fullspeed bus bandwidth must be available.
1432 * tt_available scheduling guarantees 10+% for control/bulk.
1433 */
1434 uf = uframe & 7;
1435 if (!tt_available(ehci, &stream->ps, tt, frame, uf))
1436 return 0;
1437 #else
1438 /* tt must be idle for start(s), any gap, and csplit.
1439 * assume scheduling slop leaves 10+% for control/bulk.
1440 */
1441 if (!tt_no_collision(ehci, stream->ps.bw_period,
1442 stream->ps.udev, frame, mask))
1443 return 0;
1444 #endif
1445
1446 do {
1447 unsigned max_used;
1448 unsigned i;
1449
1450 /* check starts (OUT uses more than one) */
1451 uf = uframe;
1452 max_used = ehci->uframe_periodic_max - stream->ps.usecs;
1453 for (tmp = stream->ps.cs_mask & 0xff; tmp; tmp >>= 1, uf++) {
1454 if (ehci->bandwidth[uf] > max_used)
1455 return 0;
1456 }
1457
1458 /* for IN, check CSPLIT */
1459 if (stream->ps.c_usecs) {
1460 max_used = ehci->uframe_periodic_max -
1461 stream->ps.c_usecs;
1462 uf = uframe & ~7;
1463 tmp = 1 << (2+8);
1464 for (i = (uframe & 7) + 2; i < 8; (++i, tmp <<= 1)) {
1465 if ((stream->ps.cs_mask & tmp) == 0)
1466 continue;
1467 if (ehci->bandwidth[uf+i] > max_used)
1468 return 0;
1469 }
1470 }
1471
1472 uframe += stream->ps.bw_uperiod;
1473 } while (uframe < EHCI_BANDWIDTH_SIZE);
1474
1475 stream->ps.cs_mask <<= uframe & 7;
1476 stream->splits = cpu_to_hc32(ehci, stream->ps.cs_mask);
1477 return 1;
1478 }
1479
1480 /*
1481 * This scheduler plans almost as far into the future as it has actual
1482 * periodic schedule slots. (Affected by TUNE_FLS, which defaults to
1483 * "as small as possible" to be cache-friendlier.) That limits the size
1484 * transfers you can stream reliably; avoid more than 64 msec per urb.
1485 * Also avoid queue depths of less than ehci's worst irq latency (affected
1486 * by the per-urb URB_NO_INTERRUPT hint, the log2_irq_thresh module parameter,
1487 * and other factors); or more than about 230 msec total (for portability,
1488 * given EHCI_TUNE_FLS and the slop). Or, write a smarter scheduler!
1489 */
1490
1491 static int
iso_stream_schedule(struct ehci_hcd * ehci,struct urb * urb,struct ehci_iso_stream * stream)1492 iso_stream_schedule(
1493 struct ehci_hcd *ehci,
1494 struct urb *urb,
1495 struct ehci_iso_stream *stream
1496 )
1497 {
1498 u32 now, base, next, start, period, span, now2;
1499 u32 wrap = 0, skip = 0;
1500 int status = 0;
1501 unsigned mod = ehci->periodic_size << 3;
1502 struct ehci_iso_sched *sched = urb->hcpriv;
1503 bool empty = list_empty(&stream->td_list);
1504 bool new_stream = false;
1505
1506 period = stream->uperiod;
1507 span = sched->span;
1508 if (!stream->highspeed)
1509 span <<= 3;
1510
1511 /* Start a new isochronous stream? */
1512 if (unlikely(empty && !hcd_periodic_completion_in_progress(
1513 ehci_to_hcd(ehci), urb->ep))) {
1514
1515 /* Schedule the endpoint */
1516 if (stream->ps.phase == NO_FRAME) {
1517 int done = 0;
1518 struct ehci_tt *tt = find_tt(stream->ps.udev);
1519
1520 if (IS_ERR(tt)) {
1521 status = PTR_ERR(tt);
1522 goto fail;
1523 }
1524 compute_tt_budget(ehci->tt_budget, tt);
1525
1526 start = ((-(++ehci->random_frame)) << 3) & (period - 1);
1527
1528 /* find a uframe slot with enough bandwidth.
1529 * Early uframes are more precious because full-speed
1530 * iso IN transfers can't use late uframes,
1531 * and therefore they should be allocated last.
1532 */
1533 next = start;
1534 start += period;
1535 do {
1536 start--;
1537 /* check schedule: enough space? */
1538 if (stream->highspeed) {
1539 if (itd_slot_ok(ehci, stream, start))
1540 done = 1;
1541 } else {
1542 if ((start % 8) >= 6)
1543 continue;
1544 if (sitd_slot_ok(ehci, stream, start,
1545 sched, tt))
1546 done = 1;
1547 }
1548 } while (start > next && !done);
1549
1550 /* no room in the schedule */
1551 if (!done) {
1552 ehci_dbg(ehci, "iso sched full %p", urb);
1553 status = -ENOSPC;
1554 goto fail;
1555 }
1556 stream->ps.phase = (start >> 3) &
1557 (stream->ps.period - 1);
1558 stream->ps.bw_phase = stream->ps.phase &
1559 (stream->ps.bw_period - 1);
1560 stream->ps.phase_uf = start & 7;
1561 reserve_release_iso_bandwidth(ehci, stream, 1);
1562 }
1563
1564 /* New stream is already scheduled; use the upcoming slot */
1565 else {
1566 start = (stream->ps.phase << 3) + stream->ps.phase_uf;
1567 }
1568
1569 stream->next_uframe = start;
1570 new_stream = true;
1571 }
1572
1573 now = ehci_read_frame_index(ehci) & (mod - 1);
1574
1575 /* Take the isochronous scheduling threshold into account */
1576 if (ehci->i_thresh)
1577 next = now + ehci->i_thresh; /* uframe cache */
1578 else
1579 next = (now + 2 + 7) & ~0x07; /* full frame cache */
1580
1581 /* If needed, initialize last_iso_frame so that this URB will be seen */
1582 if (ehci->isoc_count == 0)
1583 ehci->last_iso_frame = now >> 3;
1584
1585 /*
1586 * Use ehci->last_iso_frame as the base. There can't be any
1587 * TDs scheduled for earlier than that.
1588 */
1589 base = ehci->last_iso_frame << 3;
1590 next = (next - base) & (mod - 1);
1591 start = (stream->next_uframe - base) & (mod - 1);
1592
1593 if (unlikely(new_stream))
1594 goto do_ASAP;
1595
1596 /*
1597 * Typical case: reuse current schedule, stream may still be active.
1598 * Hopefully there are no gaps from the host falling behind
1599 * (irq delays etc). If there are, the behavior depends on
1600 * whether URB_ISO_ASAP is set.
1601 */
1602 now2 = (now - base) & (mod - 1);
1603
1604 /* Is the schedule about to wrap around? */
1605 if (unlikely(!empty && start < period)) {
1606 ehci_dbg(ehci, "request %p would overflow (%u-%u < %u mod %u)\n",
1607 urb, stream->next_uframe, base, period, mod);
1608 status = -EFBIG;
1609 goto fail;
1610 }
1611
1612 /* Is the next packet scheduled after the base time? */
1613 if (likely(!empty || start <= now2 + period)) {
1614
1615 /* URB_ISO_ASAP: make sure that start >= next */
1616 if (unlikely(start < next &&
1617 (urb->transfer_flags & URB_ISO_ASAP)))
1618 goto do_ASAP;
1619
1620 /* Otherwise use start, if it's not in the past */
1621 if (likely(start >= now2))
1622 goto use_start;
1623
1624 /* Otherwise we got an underrun while the queue was empty */
1625 } else {
1626 if (urb->transfer_flags & URB_ISO_ASAP)
1627 goto do_ASAP;
1628 wrap = mod;
1629 now2 += mod;
1630 }
1631
1632 /* How many uframes and packets do we need to skip? */
1633 skip = (now2 - start + period - 1) & -period;
1634 if (skip >= span) { /* Entirely in the past? */
1635 ehci_dbg(ehci, "iso underrun %p (%u+%u < %u) [%u]\n",
1636 urb, start + base, span - period, now2 + base,
1637 base);
1638
1639 /* Try to keep the last TD intact for scanning later */
1640 skip = span - period;
1641
1642 /* Will it come before the current scan position? */
1643 if (empty) {
1644 skip = span; /* Skip the entire URB */
1645 status = 1; /* and give it back immediately */
1646 iso_sched_free(stream, sched);
1647 sched = NULL;
1648 }
1649 }
1650 urb->error_count = skip / period;
1651 if (sched)
1652 sched->first_packet = urb->error_count;
1653 goto use_start;
1654
1655 do_ASAP:
1656 /* Use the first slot after "next" */
1657 start = next + ((start - next) & (period - 1));
1658
1659 use_start:
1660 /* Tried to schedule too far into the future? */
1661 if (unlikely(start + span - period >= mod + wrap)) {
1662 ehci_dbg(ehci, "request %p would overflow (%u+%u >= %u)\n",
1663 urb, start, span - period, mod + wrap);
1664 status = -EFBIG;
1665 goto fail;
1666 }
1667
1668 start += base;
1669 stream->next_uframe = (start + skip) & (mod - 1);
1670
1671 /* report high speed start in uframes; full speed, in frames */
1672 urb->start_frame = start & (mod - 1);
1673 if (!stream->highspeed)
1674 urb->start_frame >>= 3;
1675 return status;
1676
1677 fail:
1678 iso_sched_free(stream, sched);
1679 urb->hcpriv = NULL;
1680 return status;
1681 }
1682
1683 /*-------------------------------------------------------------------------*/
1684
1685 static inline void
itd_init(struct ehci_hcd * ehci,struct ehci_iso_stream * stream,struct ehci_itd * itd)1686 itd_init(struct ehci_hcd *ehci, struct ehci_iso_stream *stream,
1687 struct ehci_itd *itd)
1688 {
1689 int i;
1690
1691 /* it's been recently zeroed */
1692 itd->hw_next = EHCI_LIST_END(ehci);
1693 itd->hw_bufp[0] = stream->buf0;
1694 itd->hw_bufp[1] = stream->buf1;
1695 itd->hw_bufp[2] = stream->buf2;
1696
1697 for (i = 0; i < 8; i++)
1698 itd->index[i] = -1;
1699
1700 /* All other fields are filled when scheduling */
1701 }
1702
1703 static inline void
itd_patch(struct ehci_hcd * ehci,struct ehci_itd * itd,struct ehci_iso_sched * iso_sched,unsigned index,u16 uframe)1704 itd_patch(
1705 struct ehci_hcd *ehci,
1706 struct ehci_itd *itd,
1707 struct ehci_iso_sched *iso_sched,
1708 unsigned index,
1709 u16 uframe
1710 )
1711 {
1712 struct ehci_iso_packet *uf = &iso_sched->packet[index];
1713 unsigned pg = itd->pg;
1714
1715 /* BUG_ON(pg == 6 && uf->cross); */
1716
1717 uframe &= 0x07;
1718 itd->index[uframe] = index;
1719
1720 itd->hw_transaction[uframe] = uf->transaction;
1721 itd->hw_transaction[uframe] |= cpu_to_hc32(ehci, pg << 12);
1722 itd->hw_bufp[pg] |= cpu_to_hc32(ehci, uf->bufp & ~(u32)0);
1723 itd->hw_bufp_hi[pg] |= cpu_to_hc32(ehci, (u32)(uf->bufp >> 32));
1724
1725 /* iso_frame_desc[].offset must be strictly increasing */
1726 if (unlikely(uf->cross)) {
1727 u64 bufp = uf->bufp + 4096;
1728
1729 itd->pg = ++pg;
1730 itd->hw_bufp[pg] |= cpu_to_hc32(ehci, bufp & ~(u32)0);
1731 itd->hw_bufp_hi[pg] |= cpu_to_hc32(ehci, (u32)(bufp >> 32));
1732 }
1733 }
1734
1735 static inline void
itd_link(struct ehci_hcd * ehci,unsigned frame,struct ehci_itd * itd)1736 itd_link(struct ehci_hcd *ehci, unsigned frame, struct ehci_itd *itd)
1737 {
1738 union ehci_shadow *prev = &ehci->pshadow[frame];
1739 __hc32 *hw_p = &ehci->periodic[frame];
1740 union ehci_shadow here = *prev;
1741 __hc32 type = 0;
1742
1743 /* skip any iso nodes which might belong to previous microframes */
1744 while (here.ptr) {
1745 type = Q_NEXT_TYPE(ehci, *hw_p);
1746 if (type == cpu_to_hc32(ehci, Q_TYPE_QH))
1747 break;
1748 prev = periodic_next_shadow(ehci, prev, type);
1749 hw_p = shadow_next_periodic(ehci, &here, type);
1750 here = *prev;
1751 }
1752
1753 itd->itd_next = here;
1754 itd->hw_next = *hw_p;
1755 prev->itd = itd;
1756 itd->frame = frame;
1757 wmb();
1758 *hw_p = cpu_to_hc32(ehci, itd->itd_dma | Q_TYPE_ITD);
1759 }
1760
1761 /* fit urb's itds into the selected schedule slot; activate as needed */
itd_link_urb(struct ehci_hcd * ehci,struct urb * urb,unsigned mod,struct ehci_iso_stream * stream)1762 static void itd_link_urb(
1763 struct ehci_hcd *ehci,
1764 struct urb *urb,
1765 unsigned mod,
1766 struct ehci_iso_stream *stream
1767 )
1768 {
1769 int packet;
1770 unsigned next_uframe, uframe, frame;
1771 struct ehci_iso_sched *iso_sched = urb->hcpriv;
1772 struct ehci_itd *itd;
1773
1774 next_uframe = stream->next_uframe & (mod - 1);
1775
1776 if (unlikely(list_empty(&stream->td_list)))
1777 ehci_to_hcd(ehci)->self.bandwidth_allocated
1778 += stream->bandwidth;
1779
1780 if (ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs == 0) {
1781 if (ehci->amd_pll_fix == 1)
1782 usb_amd_quirk_pll_disable();
1783 }
1784
1785 ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs++;
1786
1787 /* fill iTDs uframe by uframe */
1788 for (packet = iso_sched->first_packet, itd = NULL;
1789 packet < urb->number_of_packets;) {
1790 if (itd == NULL) {
1791 /* ASSERT: we have all necessary itds */
1792 /* BUG_ON(list_empty(&iso_sched->td_list)); */
1793
1794 /* ASSERT: no itds for this endpoint in this uframe */
1795
1796 itd = list_entry(iso_sched->td_list.next,
1797 struct ehci_itd, itd_list);
1798 list_move_tail(&itd->itd_list, &stream->td_list);
1799 itd->stream = stream;
1800 itd->urb = urb;
1801 itd_init(ehci, stream, itd);
1802 }
1803
1804 uframe = next_uframe & 0x07;
1805 frame = next_uframe >> 3;
1806
1807 itd_patch(ehci, itd, iso_sched, packet, uframe);
1808
1809 next_uframe += stream->uperiod;
1810 next_uframe &= mod - 1;
1811 packet++;
1812
1813 /* link completed itds into the schedule */
1814 if (((next_uframe >> 3) != frame)
1815 || packet == urb->number_of_packets) {
1816 itd_link(ehci, frame & (ehci->periodic_size - 1), itd);
1817 itd = NULL;
1818 }
1819 }
1820 stream->next_uframe = next_uframe;
1821
1822 /* don't need that schedule data any more */
1823 iso_sched_free(stream, iso_sched);
1824 urb->hcpriv = stream;
1825
1826 ++ehci->isoc_count;
1827 enable_periodic(ehci);
1828 }
1829
1830 #define ISO_ERRS (EHCI_ISOC_BUF_ERR | EHCI_ISOC_BABBLE | EHCI_ISOC_XACTERR)
1831
1832 /* Process and recycle a completed ITD. Return true iff its urb completed,
1833 * and hence its completion callback probably added things to the hardware
1834 * schedule.
1835 *
1836 * Note that we carefully avoid recycling this descriptor until after any
1837 * completion callback runs, so that it won't be reused quickly. That is,
1838 * assuming (a) no more than two urbs per frame on this endpoint, and also
1839 * (b) only this endpoint's completions submit URBs. It seems some silicon
1840 * corrupts things if you reuse completed descriptors very quickly...
1841 */
itd_complete(struct ehci_hcd * ehci,struct ehci_itd * itd)1842 static bool itd_complete(struct ehci_hcd *ehci, struct ehci_itd *itd)
1843 {
1844 struct urb *urb = itd->urb;
1845 struct usb_iso_packet_descriptor *desc;
1846 u32 t;
1847 unsigned uframe;
1848 int urb_index = -1;
1849 struct ehci_iso_stream *stream = itd->stream;
1850 struct usb_device *dev;
1851 bool retval = false;
1852
1853 /* for each uframe with a packet */
1854 for (uframe = 0; uframe < 8; uframe++) {
1855 if (likely(itd->index[uframe] == -1))
1856 continue;
1857 urb_index = itd->index[uframe];
1858 desc = &urb->iso_frame_desc[urb_index];
1859
1860 t = hc32_to_cpup(ehci, &itd->hw_transaction[uframe]);
1861 itd->hw_transaction[uframe] = 0;
1862
1863 /* report transfer status */
1864 if (unlikely(t & ISO_ERRS)) {
1865 urb->error_count++;
1866 if (t & EHCI_ISOC_BUF_ERR)
1867 desc->status = usb_pipein(urb->pipe)
1868 ? -ENOSR /* hc couldn't read */
1869 : -ECOMM; /* hc couldn't write */
1870 else if (t & EHCI_ISOC_BABBLE)
1871 desc->status = -EOVERFLOW;
1872 else /* (t & EHCI_ISOC_XACTERR) */
1873 desc->status = -EPROTO;
1874
1875 /* HC need not update length with this error */
1876 if (!(t & EHCI_ISOC_BABBLE)) {
1877 desc->actual_length = EHCI_ITD_LENGTH(t);
1878 urb->actual_length += desc->actual_length;
1879 }
1880 } else if (likely((t & EHCI_ISOC_ACTIVE) == 0)) {
1881 desc->status = 0;
1882 desc->actual_length = EHCI_ITD_LENGTH(t);
1883 urb->actual_length += desc->actual_length;
1884 } else {
1885 /* URB was too late */
1886 urb->error_count++;
1887 }
1888 }
1889
1890 /* handle completion now? */
1891 if (likely((urb_index + 1) != urb->number_of_packets))
1892 goto done;
1893
1894 /*
1895 * ASSERT: it's really the last itd for this urb
1896 * list_for_each_entry (itd, &stream->td_list, itd_list)
1897 * BUG_ON(itd->urb == urb);
1898 */
1899
1900 /* give urb back to the driver; completion often (re)submits */
1901 dev = urb->dev;
1902 ehci_urb_done(ehci, urb, 0);
1903 retval = true;
1904 urb = NULL;
1905
1906 --ehci->isoc_count;
1907 disable_periodic(ehci);
1908
1909 ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs--;
1910 if (ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs == 0) {
1911 if (ehci->amd_pll_fix == 1)
1912 usb_amd_quirk_pll_enable();
1913 }
1914
1915 if (unlikely(list_is_singular(&stream->td_list)))
1916 ehci_to_hcd(ehci)->self.bandwidth_allocated
1917 -= stream->bandwidth;
1918
1919 done:
1920 itd->urb = NULL;
1921
1922 /* Add to the end of the free list for later reuse */
1923 list_move_tail(&itd->itd_list, &stream->free_list);
1924
1925 /* Recycle the iTDs when the pipeline is empty (ep no longer in use) */
1926 if (list_empty(&stream->td_list)) {
1927 list_splice_tail_init(&stream->free_list,
1928 &ehci->cached_itd_list);
1929 start_free_itds(ehci);
1930 }
1931
1932 return retval;
1933 }
1934
1935 /*-------------------------------------------------------------------------*/
1936
itd_submit(struct ehci_hcd * ehci,struct urb * urb,gfp_t mem_flags)1937 static int itd_submit(struct ehci_hcd *ehci, struct urb *urb,
1938 gfp_t mem_flags)
1939 {
1940 int status = -EINVAL;
1941 unsigned long flags;
1942 struct ehci_iso_stream *stream;
1943
1944 /* Get iso_stream head */
1945 stream = iso_stream_find(ehci, urb);
1946 if (unlikely(stream == NULL)) {
1947 ehci_dbg(ehci, "can't get iso stream\n");
1948 return -ENOMEM;
1949 }
1950 if (unlikely(urb->interval != stream->uperiod)) {
1951 ehci_dbg(ehci, "can't change iso interval %d --> %d\n",
1952 stream->uperiod, urb->interval);
1953 goto done;
1954 }
1955
1956 #ifdef EHCI_URB_TRACE
1957 ehci_dbg(ehci,
1958 "%s %s urb %p ep%d%s len %d, %d pkts %d uframes [%p]\n",
1959 __func__, urb->dev->devpath, urb,
1960 usb_pipeendpoint(urb->pipe),
1961 usb_pipein(urb->pipe) ? "in" : "out",
1962 urb->transfer_buffer_length,
1963 urb->number_of_packets, urb->interval,
1964 stream);
1965 #endif
1966
1967 /* allocate ITDs w/o locking anything */
1968 status = itd_urb_transaction(stream, ehci, urb, mem_flags);
1969 if (unlikely(status < 0)) {
1970 ehci_dbg(ehci, "can't init itds\n");
1971 goto done;
1972 }
1973
1974 /* schedule ... need to lock */
1975 spin_lock_irqsave(&ehci->lock, flags);
1976 if (unlikely(!HCD_HW_ACCESSIBLE(ehci_to_hcd(ehci)))) {
1977 status = -ESHUTDOWN;
1978 goto done_not_linked;
1979 }
1980 status = usb_hcd_link_urb_to_ep(ehci_to_hcd(ehci), urb);
1981 if (unlikely(status))
1982 goto done_not_linked;
1983 status = iso_stream_schedule(ehci, urb, stream);
1984 if (likely(status == 0)) {
1985 itd_link_urb(ehci, urb, ehci->periodic_size << 3, stream);
1986 } else if (status > 0) {
1987 status = 0;
1988 ehci_urb_done(ehci, urb, 0);
1989 } else {
1990 usb_hcd_unlink_urb_from_ep(ehci_to_hcd(ehci), urb);
1991 }
1992 done_not_linked:
1993 spin_unlock_irqrestore(&ehci->lock, flags);
1994 done:
1995 return status;
1996 }
1997
1998 /*-------------------------------------------------------------------------*/
1999
2000 /*
2001 * "Split ISO TDs" ... used for USB 1.1 devices going through the
2002 * TTs in USB 2.0 hubs. These need microframe scheduling.
2003 */
2004
2005 static inline void
sitd_sched_init(struct ehci_hcd * ehci,struct ehci_iso_sched * iso_sched,struct ehci_iso_stream * stream,struct urb * urb)2006 sitd_sched_init(
2007 struct ehci_hcd *ehci,
2008 struct ehci_iso_sched *iso_sched,
2009 struct ehci_iso_stream *stream,
2010 struct urb *urb
2011 )
2012 {
2013 unsigned i;
2014 dma_addr_t dma = urb->transfer_dma;
2015
2016 /* how many frames are needed for these transfers */
2017 iso_sched->span = urb->number_of_packets * stream->ps.period;
2018
2019 /* figure out per-frame sitd fields that we'll need later
2020 * when we fit new sitds into the schedule.
2021 */
2022 for (i = 0; i < urb->number_of_packets; i++) {
2023 struct ehci_iso_packet *packet = &iso_sched->packet[i];
2024 unsigned length;
2025 dma_addr_t buf;
2026 u32 trans;
2027
2028 length = urb->iso_frame_desc[i].length & 0x03ff;
2029 buf = dma + urb->iso_frame_desc[i].offset;
2030
2031 trans = SITD_STS_ACTIVE;
2032 if (((i + 1) == urb->number_of_packets)
2033 && !(urb->transfer_flags & URB_NO_INTERRUPT))
2034 trans |= SITD_IOC;
2035 trans |= length << 16;
2036 packet->transaction = cpu_to_hc32(ehci, trans);
2037
2038 /* might need to cross a buffer page within a td */
2039 packet->bufp = buf;
2040 packet->buf1 = (buf + length) & ~0x0fff;
2041 if (packet->buf1 != (buf & ~(u64)0x0fff))
2042 packet->cross = 1;
2043
2044 /* OUT uses multiple start-splits */
2045 if (stream->bEndpointAddress & USB_DIR_IN)
2046 continue;
2047 length = (length + 187) / 188;
2048 if (length > 1) /* BEGIN vs ALL */
2049 length |= 1 << 3;
2050 packet->buf1 |= length;
2051 }
2052 }
2053
2054 static int
sitd_urb_transaction(struct ehci_iso_stream * stream,struct ehci_hcd * ehci,struct urb * urb,gfp_t mem_flags)2055 sitd_urb_transaction(
2056 struct ehci_iso_stream *stream,
2057 struct ehci_hcd *ehci,
2058 struct urb *urb,
2059 gfp_t mem_flags
2060 )
2061 {
2062 struct ehci_sitd *sitd;
2063 dma_addr_t sitd_dma;
2064 int i;
2065 struct ehci_iso_sched *iso_sched;
2066 unsigned long flags;
2067
2068 iso_sched = iso_sched_alloc(urb->number_of_packets, mem_flags);
2069 if (iso_sched == NULL)
2070 return -ENOMEM;
2071
2072 sitd_sched_init(ehci, iso_sched, stream, urb);
2073
2074 /* allocate/init sITDs */
2075 spin_lock_irqsave(&ehci->lock, flags);
2076 for (i = 0; i < urb->number_of_packets; i++) {
2077
2078 /* NOTE: for now, we don't try to handle wraparound cases
2079 * for IN (using sitd->hw_backpointer, like a FSTN), which
2080 * means we never need two sitds for full speed packets.
2081 */
2082
2083 /*
2084 * Use siTDs from the free list, but not siTDs that may
2085 * still be in use by the hardware.
2086 */
2087 if (likely(!list_empty(&stream->free_list))) {
2088 sitd = list_first_entry(&stream->free_list,
2089 struct ehci_sitd, sitd_list);
2090 if (sitd->frame == ehci->now_frame)
2091 goto alloc_sitd;
2092 list_del(&sitd->sitd_list);
2093 sitd_dma = sitd->sitd_dma;
2094 } else {
2095 alloc_sitd:
2096 spin_unlock_irqrestore(&ehci->lock, flags);
2097 sitd = dma_pool_alloc(ehci->sitd_pool, mem_flags,
2098 &sitd_dma);
2099 spin_lock_irqsave(&ehci->lock, flags);
2100 if (!sitd) {
2101 iso_sched_free(stream, iso_sched);
2102 spin_unlock_irqrestore(&ehci->lock, flags);
2103 return -ENOMEM;
2104 }
2105 }
2106
2107 memset(sitd, 0, sizeof(*sitd));
2108 sitd->sitd_dma = sitd_dma;
2109 sitd->frame = NO_FRAME;
2110 list_add(&sitd->sitd_list, &iso_sched->td_list);
2111 }
2112
2113 /* temporarily store schedule info in hcpriv */
2114 urb->hcpriv = iso_sched;
2115 urb->error_count = 0;
2116
2117 spin_unlock_irqrestore(&ehci->lock, flags);
2118 return 0;
2119 }
2120
2121 /*-------------------------------------------------------------------------*/
2122
2123 static inline void
sitd_patch(struct ehci_hcd * ehci,struct ehci_iso_stream * stream,struct ehci_sitd * sitd,struct ehci_iso_sched * iso_sched,unsigned index)2124 sitd_patch(
2125 struct ehci_hcd *ehci,
2126 struct ehci_iso_stream *stream,
2127 struct ehci_sitd *sitd,
2128 struct ehci_iso_sched *iso_sched,
2129 unsigned index
2130 )
2131 {
2132 struct ehci_iso_packet *uf = &iso_sched->packet[index];
2133 u64 bufp;
2134
2135 sitd->hw_next = EHCI_LIST_END(ehci);
2136 sitd->hw_fullspeed_ep = stream->address;
2137 sitd->hw_uframe = stream->splits;
2138 sitd->hw_results = uf->transaction;
2139 sitd->hw_backpointer = EHCI_LIST_END(ehci);
2140
2141 bufp = uf->bufp;
2142 sitd->hw_buf[0] = cpu_to_hc32(ehci, bufp);
2143 sitd->hw_buf_hi[0] = cpu_to_hc32(ehci, bufp >> 32);
2144
2145 sitd->hw_buf[1] = cpu_to_hc32(ehci, uf->buf1);
2146 if (uf->cross)
2147 bufp += 4096;
2148 sitd->hw_buf_hi[1] = cpu_to_hc32(ehci, bufp >> 32);
2149 sitd->index = index;
2150 }
2151
2152 static inline void
sitd_link(struct ehci_hcd * ehci,unsigned frame,struct ehci_sitd * sitd)2153 sitd_link(struct ehci_hcd *ehci, unsigned frame, struct ehci_sitd *sitd)
2154 {
2155 /* note: sitd ordering could matter (CSPLIT then SSPLIT) */
2156 sitd->sitd_next = ehci->pshadow[frame];
2157 sitd->hw_next = ehci->periodic[frame];
2158 ehci->pshadow[frame].sitd = sitd;
2159 sitd->frame = frame;
2160 wmb();
2161 ehci->periodic[frame] = cpu_to_hc32(ehci, sitd->sitd_dma | Q_TYPE_SITD);
2162 }
2163
2164 /* fit urb's sitds into the selected schedule slot; activate as needed */
sitd_link_urb(struct ehci_hcd * ehci,struct urb * urb,unsigned mod,struct ehci_iso_stream * stream)2165 static void sitd_link_urb(
2166 struct ehci_hcd *ehci,
2167 struct urb *urb,
2168 unsigned mod,
2169 struct ehci_iso_stream *stream
2170 )
2171 {
2172 int packet;
2173 unsigned next_uframe;
2174 struct ehci_iso_sched *sched = urb->hcpriv;
2175 struct ehci_sitd *sitd;
2176
2177 next_uframe = stream->next_uframe;
2178
2179 if (list_empty(&stream->td_list))
2180 /* usbfs ignores TT bandwidth */
2181 ehci_to_hcd(ehci)->self.bandwidth_allocated
2182 += stream->bandwidth;
2183
2184 if (ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs == 0) {
2185 if (ehci->amd_pll_fix == 1)
2186 usb_amd_quirk_pll_disable();
2187 }
2188
2189 ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs++;
2190
2191 /* fill sITDs frame by frame */
2192 for (packet = sched->first_packet, sitd = NULL;
2193 packet < urb->number_of_packets;
2194 packet++) {
2195
2196 /* ASSERT: we have all necessary sitds */
2197 BUG_ON(list_empty(&sched->td_list));
2198
2199 /* ASSERT: no itds for this endpoint in this frame */
2200
2201 sitd = list_entry(sched->td_list.next,
2202 struct ehci_sitd, sitd_list);
2203 list_move_tail(&sitd->sitd_list, &stream->td_list);
2204 sitd->stream = stream;
2205 sitd->urb = urb;
2206
2207 sitd_patch(ehci, stream, sitd, sched, packet);
2208 sitd_link(ehci, (next_uframe >> 3) & (ehci->periodic_size - 1),
2209 sitd);
2210
2211 next_uframe += stream->uperiod;
2212 }
2213 stream->next_uframe = next_uframe & (mod - 1);
2214
2215 /* don't need that schedule data any more */
2216 iso_sched_free(stream, sched);
2217 urb->hcpriv = stream;
2218
2219 ++ehci->isoc_count;
2220 enable_periodic(ehci);
2221 }
2222
2223 /*-------------------------------------------------------------------------*/
2224
2225 #define SITD_ERRS (SITD_STS_ERR | SITD_STS_DBE | SITD_STS_BABBLE \
2226 | SITD_STS_XACT | SITD_STS_MMF)
2227
2228 /* Process and recycle a completed SITD. Return true iff its urb completed,
2229 * and hence its completion callback probably added things to the hardware
2230 * schedule.
2231 *
2232 * Note that we carefully avoid recycling this descriptor until after any
2233 * completion callback runs, so that it won't be reused quickly. That is,
2234 * assuming (a) no more than two urbs per frame on this endpoint, and also
2235 * (b) only this endpoint's completions submit URBs. It seems some silicon
2236 * corrupts things if you reuse completed descriptors very quickly...
2237 */
sitd_complete(struct ehci_hcd * ehci,struct ehci_sitd * sitd)2238 static bool sitd_complete(struct ehci_hcd *ehci, struct ehci_sitd *sitd)
2239 {
2240 struct urb *urb = sitd->urb;
2241 struct usb_iso_packet_descriptor *desc;
2242 u32 t;
2243 int urb_index;
2244 struct ehci_iso_stream *stream = sitd->stream;
2245 struct usb_device *dev;
2246 bool retval = false;
2247
2248 urb_index = sitd->index;
2249 desc = &urb->iso_frame_desc[urb_index];
2250 t = hc32_to_cpup(ehci, &sitd->hw_results);
2251
2252 /* report transfer status */
2253 if (unlikely(t & SITD_ERRS)) {
2254 urb->error_count++;
2255 if (t & SITD_STS_DBE)
2256 desc->status = usb_pipein(urb->pipe)
2257 ? -ENOSR /* hc couldn't read */
2258 : -ECOMM; /* hc couldn't write */
2259 else if (t & SITD_STS_BABBLE)
2260 desc->status = -EOVERFLOW;
2261 else /* XACT, MMF, etc */
2262 desc->status = -EPROTO;
2263 } else if (unlikely(t & SITD_STS_ACTIVE)) {
2264 /* URB was too late */
2265 urb->error_count++;
2266 } else {
2267 desc->status = 0;
2268 desc->actual_length = desc->length - SITD_LENGTH(t);
2269 urb->actual_length += desc->actual_length;
2270 }
2271
2272 /* handle completion now? */
2273 if ((urb_index + 1) != urb->number_of_packets)
2274 goto done;
2275
2276 /*
2277 * ASSERT: it's really the last sitd for this urb
2278 * list_for_each_entry (sitd, &stream->td_list, sitd_list)
2279 * BUG_ON(sitd->urb == urb);
2280 */
2281
2282 /* give urb back to the driver; completion often (re)submits */
2283 dev = urb->dev;
2284 ehci_urb_done(ehci, urb, 0);
2285 retval = true;
2286 urb = NULL;
2287
2288 --ehci->isoc_count;
2289 disable_periodic(ehci);
2290
2291 ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs--;
2292 if (ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs == 0) {
2293 if (ehci->amd_pll_fix == 1)
2294 usb_amd_quirk_pll_enable();
2295 }
2296
2297 if (list_is_singular(&stream->td_list))
2298 ehci_to_hcd(ehci)->self.bandwidth_allocated
2299 -= stream->bandwidth;
2300
2301 done:
2302 sitd->urb = NULL;
2303
2304 /* Add to the end of the free list for later reuse */
2305 list_move_tail(&sitd->sitd_list, &stream->free_list);
2306
2307 /* Recycle the siTDs when the pipeline is empty (ep no longer in use) */
2308 if (list_empty(&stream->td_list)) {
2309 list_splice_tail_init(&stream->free_list,
2310 &ehci->cached_sitd_list);
2311 start_free_itds(ehci);
2312 }
2313
2314 return retval;
2315 }
2316
2317
sitd_submit(struct ehci_hcd * ehci,struct urb * urb,gfp_t mem_flags)2318 static int sitd_submit(struct ehci_hcd *ehci, struct urb *urb,
2319 gfp_t mem_flags)
2320 {
2321 int status = -EINVAL;
2322 unsigned long flags;
2323 struct ehci_iso_stream *stream;
2324
2325 /* Get iso_stream head */
2326 stream = iso_stream_find(ehci, urb);
2327 if (stream == NULL) {
2328 ehci_dbg(ehci, "can't get iso stream\n");
2329 return -ENOMEM;
2330 }
2331 if (urb->interval != stream->ps.period) {
2332 ehci_dbg(ehci, "can't change iso interval %d --> %d\n",
2333 stream->ps.period, urb->interval);
2334 goto done;
2335 }
2336
2337 #ifdef EHCI_URB_TRACE
2338 ehci_dbg(ehci,
2339 "submit %p dev%s ep%d%s-iso len %d\n",
2340 urb, urb->dev->devpath,
2341 usb_pipeendpoint(urb->pipe),
2342 usb_pipein(urb->pipe) ? "in" : "out",
2343 urb->transfer_buffer_length);
2344 #endif
2345
2346 /* allocate SITDs */
2347 status = sitd_urb_transaction(stream, ehci, urb, mem_flags);
2348 if (status < 0) {
2349 ehci_dbg(ehci, "can't init sitds\n");
2350 goto done;
2351 }
2352
2353 /* schedule ... need to lock */
2354 spin_lock_irqsave(&ehci->lock, flags);
2355 if (unlikely(!HCD_HW_ACCESSIBLE(ehci_to_hcd(ehci)))) {
2356 status = -ESHUTDOWN;
2357 goto done_not_linked;
2358 }
2359 status = usb_hcd_link_urb_to_ep(ehci_to_hcd(ehci), urb);
2360 if (unlikely(status))
2361 goto done_not_linked;
2362 status = iso_stream_schedule(ehci, urb, stream);
2363 if (likely(status == 0)) {
2364 sitd_link_urb(ehci, urb, ehci->periodic_size << 3, stream);
2365 } else if (status > 0) {
2366 status = 0;
2367 ehci_urb_done(ehci, urb, 0);
2368 } else {
2369 usb_hcd_unlink_urb_from_ep(ehci_to_hcd(ehci), urb);
2370 }
2371 done_not_linked:
2372 spin_unlock_irqrestore(&ehci->lock, flags);
2373 done:
2374 return status;
2375 }
2376
2377 /*-------------------------------------------------------------------------*/
2378
scan_isoc(struct ehci_hcd * ehci)2379 static void scan_isoc(struct ehci_hcd *ehci)
2380 {
2381 unsigned uf, now_frame, frame;
2382 unsigned fmask = ehci->periodic_size - 1;
2383 bool modified, live;
2384 union ehci_shadow q, *q_p;
2385 __hc32 type, *hw_p;
2386
2387 /*
2388 * When running, scan from last scan point up to "now"
2389 * else clean up by scanning everything that's left.
2390 * Touches as few pages as possible: cache-friendly.
2391 */
2392 if (ehci->rh_state >= EHCI_RH_RUNNING) {
2393 uf = ehci_read_frame_index(ehci);
2394 now_frame = (uf >> 3) & fmask;
2395 live = true;
2396 } else {
2397 now_frame = (ehci->last_iso_frame - 1) & fmask;
2398 live = false;
2399 }
2400 ehci->now_frame = now_frame;
2401
2402 frame = ehci->last_iso_frame;
2403
2404 restart:
2405 /* Scan each element in frame's queue for completions */
2406 q_p = &ehci->pshadow[frame];
2407 hw_p = &ehci->periodic[frame];
2408 q.ptr = q_p->ptr;
2409 type = Q_NEXT_TYPE(ehci, *hw_p);
2410 modified = false;
2411
2412 while (q.ptr != NULL) {
2413 switch (hc32_to_cpu(ehci, type)) {
2414 case Q_TYPE_ITD:
2415 /*
2416 * If this ITD is still active, leave it for
2417 * later processing ... check the next entry.
2418 * No need to check for activity unless the
2419 * frame is current.
2420 */
2421 if (frame == now_frame && live) {
2422 rmb();
2423 for (uf = 0; uf < 8; uf++) {
2424 if (q.itd->hw_transaction[uf] &
2425 ITD_ACTIVE(ehci))
2426 break;
2427 }
2428 if (uf < 8) {
2429 q_p = &q.itd->itd_next;
2430 hw_p = &q.itd->hw_next;
2431 type = Q_NEXT_TYPE(ehci,
2432 q.itd->hw_next);
2433 q = *q_p;
2434 break;
2435 }
2436 }
2437
2438 /*
2439 * Take finished ITDs out of the schedule
2440 * and process them: recycle, maybe report
2441 * URB completion. HC won't cache the
2442 * pointer for much longer, if at all.
2443 */
2444 *q_p = q.itd->itd_next;
2445 if (!ehci->use_dummy_qh ||
2446 q.itd->hw_next != EHCI_LIST_END(ehci))
2447 *hw_p = q.itd->hw_next;
2448 else
2449 *hw_p = cpu_to_hc32(ehci, ehci->dummy->qh_dma);
2450 type = Q_NEXT_TYPE(ehci, q.itd->hw_next);
2451 wmb();
2452 modified = itd_complete(ehci, q.itd);
2453 q = *q_p;
2454 break;
2455 case Q_TYPE_SITD:
2456 /*
2457 * If this SITD is still active, leave it for
2458 * later processing ... check the next entry.
2459 * No need to check for activity unless the
2460 * frame is current.
2461 */
2462 if (((frame == now_frame) ||
2463 (((frame + 1) & fmask) == now_frame))
2464 && live
2465 && (q.sitd->hw_results & SITD_ACTIVE(ehci))) {
2466
2467 q_p = &q.sitd->sitd_next;
2468 hw_p = &q.sitd->hw_next;
2469 type = Q_NEXT_TYPE(ehci, q.sitd->hw_next);
2470 q = *q_p;
2471 break;
2472 }
2473
2474 /*
2475 * Take finished SITDs out of the schedule
2476 * and process them: recycle, maybe report
2477 * URB completion.
2478 */
2479 *q_p = q.sitd->sitd_next;
2480 if (!ehci->use_dummy_qh ||
2481 q.sitd->hw_next != EHCI_LIST_END(ehci))
2482 *hw_p = q.sitd->hw_next;
2483 else
2484 *hw_p = cpu_to_hc32(ehci, ehci->dummy->qh_dma);
2485 type = Q_NEXT_TYPE(ehci, q.sitd->hw_next);
2486 wmb();
2487 modified = sitd_complete(ehci, q.sitd);
2488 q = *q_p;
2489 break;
2490 default:
2491 ehci_dbg(ehci, "corrupt type %d frame %d shadow %p\n",
2492 type, frame, q.ptr);
2493 /* BUG(); */
2494 /* FALL THROUGH */
2495 case Q_TYPE_QH:
2496 case Q_TYPE_FSTN:
2497 /* End of the iTDs and siTDs */
2498 q.ptr = NULL;
2499 break;
2500 }
2501
2502 /* Assume completion callbacks modify the queue */
2503 if (unlikely(modified && ehci->isoc_count > 0))
2504 goto restart;
2505 }
2506
2507 /* Stop when we have reached the current frame */
2508 if (frame == now_frame)
2509 return;
2510
2511 /* The last frame may still have active siTDs */
2512 ehci->last_iso_frame = frame;
2513 frame = (frame + 1) & fmask;
2514
2515 goto restart;
2516 }
2517