Lines Matching refs:q
212 struct qfq_sched *q = qdisc_priv(sch); in qfq_find_class() local
215 clc = qdisc_class_find(&q->clhash, classid); in qfq_find_class()
223 unsigned int len = cl->qdisc->q.qlen; in qfq_purge_queue()
266 static void qfq_init_agg(struct qfq_sched *q, struct qfq_aggregate *agg, in qfq_init_agg() argument
270 hlist_add_head(&agg->nonfull_next, &q->nonfull_aggs); in qfq_init_agg()
276 static struct qfq_aggregate *qfq_find_agg(struct qfq_sched *q, in qfq_find_agg() argument
281 hlist_for_each_entry(agg, &q->nonfull_aggs, nonfull_next) in qfq_find_agg()
290 static void qfq_update_agg(struct qfq_sched *q, struct qfq_aggregate *agg, in qfq_update_agg() argument
295 if (new_num_classes == q->max_agg_classes) in qfq_update_agg()
299 new_num_classes == q->max_agg_classes - 1) /* agg no more full */ in qfq_update_agg()
300 hlist_add_head(&agg->nonfull_next, &q->nonfull_aggs); in qfq_update_agg()
312 q->min_slot_shift); in qfq_update_agg()
313 agg->grp = &q->groups[i]; in qfq_update_agg()
316 q->wsum += in qfq_update_agg()
318 q->iwsum = ONE_FP / q->wsum; in qfq_update_agg()
324 static void qfq_add_to_agg(struct qfq_sched *q, in qfq_add_to_agg() argument
330 qfq_update_agg(q, agg, agg->num_classes+1); in qfq_add_to_agg()
331 if (cl->qdisc->q.qlen > 0) { /* adding an active class */ in qfq_add_to_agg()
334 cl && q->in_serv_agg != agg) /* agg was inactive */ in qfq_add_to_agg()
335 qfq_activate_agg(q, agg, enqueue); /* schedule agg */ in qfq_add_to_agg()
341 static void qfq_destroy_agg(struct qfq_sched *q, struct qfq_aggregate *agg) in qfq_destroy_agg() argument
345 q->wsum -= agg->class_weight; in qfq_destroy_agg()
346 if (q->wsum != 0) in qfq_destroy_agg()
347 q->iwsum = ONE_FP / q->wsum; in qfq_destroy_agg()
349 if (q->in_serv_agg == agg) in qfq_destroy_agg()
350 q->in_serv_agg = qfq_choose_next_agg(q); in qfq_destroy_agg()
355 static void qfq_deactivate_class(struct qfq_sched *q, struct qfq_class *cl) in qfq_deactivate_class() argument
362 qfq_deactivate_agg(q, agg); in qfq_deactivate_class()
366 static void qfq_rm_from_agg(struct qfq_sched *q, struct qfq_class *cl) in qfq_rm_from_agg() argument
372 qfq_destroy_agg(q, agg); in qfq_rm_from_agg()
375 qfq_update_agg(q, agg, agg->num_classes-1); in qfq_rm_from_agg()
379 static void qfq_deact_rm_from_agg(struct qfq_sched *q, struct qfq_class *cl) in qfq_deact_rm_from_agg() argument
381 if (cl->qdisc->q.qlen > 0) /* class is active */ in qfq_deact_rm_from_agg()
382 qfq_deactivate_class(q, cl); in qfq_deact_rm_from_agg()
384 qfq_rm_from_agg(q, cl); in qfq_deact_rm_from_agg()
391 struct qfq_sched *q = qdisc_priv(sch); in qfq_change_agg() local
392 struct qfq_aggregate *new_agg = qfq_find_agg(q, lmax, weight); in qfq_change_agg()
398 qfq_init_agg(q, new_agg, lmax, weight); in qfq_change_agg()
400 qfq_deact_rm_from_agg(q, cl); in qfq_change_agg()
401 qfq_add_to_agg(q, new_agg, cl); in qfq_change_agg()
409 struct qfq_sched *q = qdisc_priv(sch); in qfq_change_class() local
455 if (q->wsum + delta_w > QFQ_MAX_WSUM) { in qfq_change_class()
457 delta_w, q->wsum); in qfq_change_class()
498 qdisc_class_hash_insert(&q->clhash, &cl->common); in qfq_change_class()
501 qdisc_class_hash_grow(sch, &q->clhash); in qfq_change_class()
505 new_agg = qfq_find_agg(q, lmax, weight); in qfq_change_class()
515 qfq_init_agg(q, new_agg, lmax, weight); in qfq_change_class()
518 qfq_deact_rm_from_agg(q, cl); in qfq_change_class()
519 qfq_add_to_agg(q, new_agg, cl); in qfq_change_class()
533 struct qfq_sched *q = qdisc_priv(sch); in qfq_destroy_class() local
535 qfq_rm_from_agg(q, cl); in qfq_destroy_class()
543 struct qfq_sched *q = qdisc_priv(sch); in qfq_delete_class() local
552 qdisc_class_hash_remove(&q->clhash, &cl->common); in qfq_delete_class()
585 struct qfq_sched *q = qdisc_priv(sch); in qfq_tcf_chain() local
590 return &q->filter_list; in qfq_tcf_chain()
671 &cl->qdisc->qstats, cl->qdisc->q.qlen) < 0) in qfq_dump_class_stats()
679 struct qfq_sched *q = qdisc_priv(sch); in qfq_walk() local
686 for (i = 0; i < q->clhash.hashsize; i++) { in qfq_walk()
687 hlist_for_each_entry(cl, &q->clhash.hash[i], common.hnode) { in qfq_walk()
704 struct qfq_sched *q = qdisc_priv(sch); in qfq_classify() local
718 fl = rcu_dereference_bh(q->filter_list); in qfq_classify()
752 static inline struct qfq_group *qfq_ffs(struct qfq_sched *q, in qfq_ffs() argument
756 return &q->groups[index]; in qfq_ffs()
769 static int qfq_calc_state(struct qfq_sched *q, const struct qfq_group *grp) in qfq_calc_state() argument
772 unsigned int state = qfq_gt(grp->S, q->V); in qfq_calc_state()
773 unsigned long mask = mask_from(q->bitmaps[ER], grp->index); in qfq_calc_state()
777 next = qfq_ffs(q, mask); in qfq_calc_state()
792 static inline void qfq_move_groups(struct qfq_sched *q, unsigned long mask, in qfq_move_groups() argument
795 q->bitmaps[dst] |= q->bitmaps[src] & mask; in qfq_move_groups()
796 q->bitmaps[src] &= ~mask; in qfq_move_groups()
799 static void qfq_unblock_groups(struct qfq_sched *q, int index, u64 old_F) in qfq_unblock_groups() argument
801 unsigned long mask = mask_from(q->bitmaps[ER], index + 1); in qfq_unblock_groups()
805 next = qfq_ffs(q, mask); in qfq_unblock_groups()
811 qfq_move_groups(q, mask, EB, ER); in qfq_unblock_groups()
812 qfq_move_groups(q, mask, IB, IR); in qfq_unblock_groups()
825 static void qfq_make_eligible(struct qfq_sched *q) in qfq_make_eligible() argument
827 unsigned long vslot = q->V >> q->min_slot_shift; in qfq_make_eligible()
828 unsigned long old_vslot = q->oldV >> q->min_slot_shift; in qfq_make_eligible()
839 qfq_move_groups(q, mask, IR, ER); in qfq_make_eligible()
840 qfq_move_groups(q, mask, IB, EB); in qfq_make_eligible()
980 static void qfq_update_eligible(struct qfq_sched *q) in qfq_update_eligible() argument
985 ineligible = q->bitmaps[IR] | q->bitmaps[IB]; in qfq_update_eligible()
987 if (!q->bitmaps[ER]) { in qfq_update_eligible()
988 grp = qfq_ffs(q, ineligible); in qfq_update_eligible()
989 if (qfq_gt(grp->S, q->V)) in qfq_update_eligible()
990 q->V = grp->S; in qfq_update_eligible()
992 qfq_make_eligible(q); in qfq_update_eligible()
1004 if (cl->qdisc->q.qlen == 0) /* no more packets, remove from list */ in agg_dequeue()
1054 static void qfq_update_start(struct qfq_sched *q, struct qfq_aggregate *agg) in qfq_update_start() argument
1061 limit = qfq_round_down(q->V, slot_shift) + (1ULL << slot_shift); in qfq_update_start()
1063 if (!qfq_gt(agg->F, q->V) || qfq_gt(roundedF, limit)) { in qfq_update_start()
1065 mask = mask_from(q->bitmaps[ER], agg->grp->index); in qfq_update_start()
1067 struct qfq_group *next = qfq_ffs(q, mask); in qfq_update_start()
1076 agg->S = q->V; in qfq_update_start()
1087 qfq_update_agg_ts(struct qfq_sched *q, in qfq_update_agg_ts() argument
1091 qfq_update_start(q, agg); in qfq_update_agg_ts()
1098 static void qfq_schedule_agg(struct qfq_sched *q, struct qfq_aggregate *agg);
1102 struct qfq_sched *q = qdisc_priv(sch); in qfq_dequeue() local
1103 struct qfq_aggregate *in_serv_agg = q->in_serv_agg; in qfq_dequeue()
1138 qfq_update_agg_ts(q, in_serv_agg, requeue); in qfq_dequeue()
1139 qfq_schedule_agg(q, in_serv_agg); in qfq_dequeue()
1140 } else if (sch->q.qlen == 0) { /* no aggregate to serve */ in qfq_dequeue()
1141 q->in_serv_agg = NULL; in qfq_dequeue()
1149 in_serv_agg = q->in_serv_agg = qfq_choose_next_agg(q); in qfq_dequeue()
1155 sch->q.qlen--; in qfq_dequeue()
1168 q->V += (u64)len * q->iwsum; in qfq_dequeue()
1171 (unsigned long long) q->V); in qfq_dequeue()
1176 static struct qfq_aggregate *qfq_choose_next_agg(struct qfq_sched *q) in qfq_choose_next_agg() argument
1182 qfq_update_eligible(q); in qfq_choose_next_agg()
1183 q->oldV = q->V; in qfq_choose_next_agg()
1185 if (!q->bitmaps[ER]) in qfq_choose_next_agg()
1188 grp = qfq_ffs(q, q->bitmaps[ER]); in qfq_choose_next_agg()
1199 __clear_bit(grp->index, &q->bitmaps[ER]); in qfq_choose_next_agg()
1209 __clear_bit(grp->index, &q->bitmaps[ER]); in qfq_choose_next_agg()
1210 s = qfq_calc_state(q, grp); in qfq_choose_next_agg()
1211 __set_bit(grp->index, &q->bitmaps[s]); in qfq_choose_next_agg()
1214 qfq_unblock_groups(q, grp->index, old_F); in qfq_choose_next_agg()
1221 struct qfq_sched *q = qdisc_priv(sch); in qfq_enqueue() local
1255 ++sch->q.qlen; in qfq_enqueue()
1259 if (cl->qdisc->q.qlen != 1) { in qfq_enqueue()
1273 q->in_serv_agg == agg) in qfq_enqueue()
1276 qfq_activate_agg(q, agg, enqueue); in qfq_enqueue()
1284 static void qfq_schedule_agg(struct qfq_sched *q, struct qfq_aggregate *agg) in qfq_schedule_agg() argument
1308 __clear_bit(grp->index, &q->bitmaps[IR]); in qfq_schedule_agg()
1309 __clear_bit(grp->index, &q->bitmaps[IB]); in qfq_schedule_agg()
1310 } else if (!q->bitmaps[ER] && qfq_gt(roundedS, q->V) && in qfq_schedule_agg()
1311 q->in_serv_agg == NULL) in qfq_schedule_agg()
1312 q->V = roundedS; in qfq_schedule_agg()
1316 s = qfq_calc_state(q, grp); in qfq_schedule_agg()
1317 __set_bit(grp->index, &q->bitmaps[s]); in qfq_schedule_agg()
1320 s, q->bitmaps[s], in qfq_schedule_agg()
1323 (unsigned long long) q->V); in qfq_schedule_agg()
1331 static void qfq_activate_agg(struct qfq_sched *q, struct qfq_aggregate *agg, in qfq_activate_agg() argument
1336 qfq_update_agg_ts(q, agg, reason); in qfq_activate_agg()
1337 if (q->in_serv_agg == NULL) { /* no aggr. in service or scheduled */ in qfq_activate_agg()
1338 q->in_serv_agg = agg; /* start serving this aggregate */ in qfq_activate_agg()
1340 q->oldV = q->V = agg->S; in qfq_activate_agg()
1341 } else if (agg != q->in_serv_agg) in qfq_activate_agg()
1342 qfq_schedule_agg(q, agg); in qfq_activate_agg()
1345 static void qfq_slot_remove(struct qfq_sched *q, struct qfq_group *grp, in qfq_slot_remove() argument
1368 static void qfq_deactivate_agg(struct qfq_sched *q, struct qfq_aggregate *agg) in qfq_deactivate_agg() argument
1375 if (agg == q->in_serv_agg) { in qfq_deactivate_agg()
1377 q->in_serv_agg = qfq_choose_next_agg(q); in qfq_deactivate_agg()
1382 qfq_slot_remove(q, grp, agg); in qfq_deactivate_agg()
1385 __clear_bit(grp->index, &q->bitmaps[IR]); in qfq_deactivate_agg()
1386 __clear_bit(grp->index, &q->bitmaps[EB]); in qfq_deactivate_agg()
1387 __clear_bit(grp->index, &q->bitmaps[IB]); in qfq_deactivate_agg()
1389 if (test_bit(grp->index, &q->bitmaps[ER]) && in qfq_deactivate_agg()
1390 !(q->bitmaps[ER] & ~((1UL << grp->index) - 1))) { in qfq_deactivate_agg()
1391 mask = q->bitmaps[ER] & ((1UL << grp->index) - 1); in qfq_deactivate_agg()
1396 qfq_move_groups(q, mask, EB, ER); in qfq_deactivate_agg()
1397 qfq_move_groups(q, mask, IB, IR); in qfq_deactivate_agg()
1399 __clear_bit(grp->index, &q->bitmaps[ER]); in qfq_deactivate_agg()
1404 __clear_bit(grp->index, &q->bitmaps[ER]); in qfq_deactivate_agg()
1405 __clear_bit(grp->index, &q->bitmaps[IR]); in qfq_deactivate_agg()
1406 __clear_bit(grp->index, &q->bitmaps[EB]); in qfq_deactivate_agg()
1407 __clear_bit(grp->index, &q->bitmaps[IB]); in qfq_deactivate_agg()
1410 s = qfq_calc_state(q, grp); in qfq_deactivate_agg()
1411 __set_bit(grp->index, &q->bitmaps[s]); in qfq_deactivate_agg()
1418 struct qfq_sched *q = qdisc_priv(sch); in qfq_qlen_notify() local
1421 if (cl->qdisc->q.qlen == 0) in qfq_qlen_notify()
1422 qfq_deactivate_class(q, cl); in qfq_qlen_notify()
1425 static unsigned int qfq_drop_from_slot(struct qfq_sched *q, in qfq_drop_from_slot() argument
1440 if (cl->qdisc->q.qlen == 0) in qfq_drop_from_slot()
1441 qfq_deactivate_class(q, cl); in qfq_drop_from_slot()
1452 struct qfq_sched *q = qdisc_priv(sch); in qfq_drop() local
1457 grp = &q->groups[i]; in qfq_drop()
1459 len = qfq_drop_from_slot(q, &grp->slots[j]); in qfq_drop()
1461 sch->q.qlen--; in qfq_drop()
1473 struct qfq_sched *q = qdisc_priv(sch); in qfq_init_qdisc() local
1478 err = qdisc_class_hash_init(&q->clhash); in qfq_init_qdisc()
1488 q->max_agg_classes = 1<<max_cl_shift; in qfq_init_qdisc()
1492 q->min_slot_shift = FRAC_BITS + maxbudg_shift - QFQ_MAX_INDEX; in qfq_init_qdisc()
1495 grp = &q->groups[i]; in qfq_init_qdisc()
1497 grp->slot_shift = q->min_slot_shift + i; in qfq_init_qdisc()
1502 INIT_HLIST_HEAD(&q->nonfull_aggs); in qfq_init_qdisc()
1509 struct qfq_sched *q = qdisc_priv(sch); in qfq_reset_qdisc() local
1513 for (i = 0; i < q->clhash.hashsize; i++) { in qfq_reset_qdisc()
1514 hlist_for_each_entry(cl, &q->clhash.hash[i], common.hnode) { in qfq_reset_qdisc()
1515 if (cl->qdisc->q.qlen > 0) in qfq_reset_qdisc()
1516 qfq_deactivate_class(q, cl); in qfq_reset_qdisc()
1521 sch->q.qlen = 0; in qfq_reset_qdisc()
1526 struct qfq_sched *q = qdisc_priv(sch); in qfq_destroy_qdisc() local
1531 tcf_destroy_chain(&q->filter_list); in qfq_destroy_qdisc()
1533 for (i = 0; i < q->clhash.hashsize; i++) { in qfq_destroy_qdisc()
1534 hlist_for_each_entry_safe(cl, next, &q->clhash.hash[i], in qfq_destroy_qdisc()
1539 qdisc_class_hash_destroy(&q->clhash); in qfq_destroy_qdisc()