Lines Matching refs:ring
151 struct mlx4_en_rx_ring *ring) in mlx4_en_init_allocator() argument
159 if (mlx4_alloc_pages(priv, &ring->page_alloc[i], in mlx4_en_init_allocator()
169 page_alloc = &ring->page_alloc[i]; in mlx4_en_init_allocator()
181 struct mlx4_en_rx_ring *ring) in mlx4_en_destroy_allocator() argument
189 page_alloc = &ring->page_alloc[i]; in mlx4_en_destroy_allocator()
205 struct mlx4_en_rx_ring *ring, int index) in mlx4_en_init_rx_desc() argument
207 struct mlx4_en_rx_desc *rx_desc = ring->buf + ring->stride * index; in mlx4_en_init_rx_desc()
221 possible_frags = (ring->stride - sizeof(struct mlx4_en_rx_desc)) / DS_SIZE; in mlx4_en_init_rx_desc()
230 struct mlx4_en_rx_ring *ring, int index, in mlx4_en_prepare_rx_desc() argument
233 struct mlx4_en_rx_desc *rx_desc = ring->buf + (index * ring->stride); in mlx4_en_prepare_rx_desc()
234 struct mlx4_en_rx_alloc *frags = ring->rx_info + in mlx4_en_prepare_rx_desc()
237 return mlx4_en_alloc_frags(priv, rx_desc, frags, ring->page_alloc, gfp); in mlx4_en_prepare_rx_desc()
240 static inline bool mlx4_en_is_ring_empty(struct mlx4_en_rx_ring *ring) in mlx4_en_is_ring_empty() argument
242 BUG_ON((u32)(ring->prod - ring->cons) > ring->actual_size); in mlx4_en_is_ring_empty()
243 return ring->prod == ring->cons; in mlx4_en_is_ring_empty()
246 static inline void mlx4_en_update_rx_prod_db(struct mlx4_en_rx_ring *ring) in mlx4_en_update_rx_prod_db() argument
248 *ring->wqres.db.db = cpu_to_be32(ring->prod & 0xffff); in mlx4_en_update_rx_prod_db()
252 struct mlx4_en_rx_ring *ring, in mlx4_en_free_rx_desc() argument
258 frags = ring->rx_info + (index << priv->log_rx_info); in mlx4_en_free_rx_desc()
267 struct mlx4_en_rx_ring *ring; in mlx4_en_fill_rx_buffers() local
274 ring = priv->rx_ring[ring_ind]; in mlx4_en_fill_rx_buffers()
276 if (mlx4_en_prepare_rx_desc(priv, ring, in mlx4_en_fill_rx_buffers()
277 ring->actual_size, in mlx4_en_fill_rx_buffers()
279 if (ring->actual_size < MLX4_EN_MIN_RX_SIZE) { in mlx4_en_fill_rx_buffers()
283 new_size = rounddown_pow_of_two(ring->actual_size); in mlx4_en_fill_rx_buffers()
285 ring->actual_size, new_size); in mlx4_en_fill_rx_buffers()
289 ring->actual_size++; in mlx4_en_fill_rx_buffers()
290 ring->prod++; in mlx4_en_fill_rx_buffers()
297 ring = priv->rx_ring[ring_ind]; in mlx4_en_fill_rx_buffers()
298 while (ring->actual_size > new_size) { in mlx4_en_fill_rx_buffers()
299 ring->actual_size--; in mlx4_en_fill_rx_buffers()
300 ring->prod--; in mlx4_en_fill_rx_buffers()
301 mlx4_en_free_rx_desc(priv, ring, ring->actual_size); in mlx4_en_fill_rx_buffers()
309 struct mlx4_en_rx_ring *ring) in mlx4_en_free_rx_buf() argument
314 ring->cons, ring->prod); in mlx4_en_free_rx_buf()
317 while (!mlx4_en_is_ring_empty(ring)) { in mlx4_en_free_rx_buf()
318 index = ring->cons & ring->size_mask; in mlx4_en_free_rx_buf()
320 mlx4_en_free_rx_desc(priv, ring, index); in mlx4_en_free_rx_buf()
321 ++ring->cons; in mlx4_en_free_rx_buf()
356 struct mlx4_en_rx_ring *ring; in mlx4_en_create_rx_ring() local
360 ring = kzalloc_node(sizeof(*ring), GFP_KERNEL, node); in mlx4_en_create_rx_ring()
361 if (!ring) { in mlx4_en_create_rx_ring()
362 ring = kzalloc(sizeof(*ring), GFP_KERNEL); in mlx4_en_create_rx_ring()
363 if (!ring) { in mlx4_en_create_rx_ring()
369 ring->prod = 0; in mlx4_en_create_rx_ring()
370 ring->cons = 0; in mlx4_en_create_rx_ring()
371 ring->size = size; in mlx4_en_create_rx_ring()
372 ring->size_mask = size - 1; in mlx4_en_create_rx_ring()
373 ring->stride = stride; in mlx4_en_create_rx_ring()
374 ring->log_stride = ffs(ring->stride) - 1; in mlx4_en_create_rx_ring()
375 ring->buf_size = ring->size * ring->stride + TXBB_SIZE; in mlx4_en_create_rx_ring()
379 ring->rx_info = vmalloc_node(tmp, node); in mlx4_en_create_rx_ring()
380 if (!ring->rx_info) { in mlx4_en_create_rx_ring()
381 ring->rx_info = vmalloc(tmp); in mlx4_en_create_rx_ring()
382 if (!ring->rx_info) { in mlx4_en_create_rx_ring()
389 ring->rx_info, tmp); in mlx4_en_create_rx_ring()
393 err = mlx4_alloc_hwq_res(mdev->dev, &ring->wqres, in mlx4_en_create_rx_ring()
394 ring->buf_size, 2 * PAGE_SIZE); in mlx4_en_create_rx_ring()
399 err = mlx4_en_map_buffer(&ring->wqres.buf); in mlx4_en_create_rx_ring()
404 ring->buf = ring->wqres.buf.direct.buf; in mlx4_en_create_rx_ring()
406 ring->hwtstamp_rx_filter = priv->hwtstamp_config.rx_filter; in mlx4_en_create_rx_ring()
408 *pring = ring; in mlx4_en_create_rx_ring()
412 mlx4_free_hwq_res(mdev->dev, &ring->wqres, ring->buf_size); in mlx4_en_create_rx_ring()
414 vfree(ring->rx_info); in mlx4_en_create_rx_ring()
415 ring->rx_info = NULL; in mlx4_en_create_rx_ring()
417 kfree(ring); in mlx4_en_create_rx_ring()
425 struct mlx4_en_rx_ring *ring; in mlx4_en_activate_rx_rings() local
433 ring = priv->rx_ring[ring_ind]; in mlx4_en_activate_rx_rings()
435 ring->prod = 0; in mlx4_en_activate_rx_rings()
436 ring->cons = 0; in mlx4_en_activate_rx_rings()
437 ring->actual_size = 0; in mlx4_en_activate_rx_rings()
438 ring->cqn = priv->rx_cq[ring_ind]->mcq.cqn; in mlx4_en_activate_rx_rings()
440 ring->stride = stride; in mlx4_en_activate_rx_rings()
441 if (ring->stride <= TXBB_SIZE) { in mlx4_en_activate_rx_rings()
443 __be32 *ptr = (__be32 *)ring->buf; in mlx4_en_activate_rx_rings()
447 ring->buf += TXBB_SIZE; in mlx4_en_activate_rx_rings()
450 ring->log_stride = ffs(ring->stride) - 1; in mlx4_en_activate_rx_rings()
451 ring->buf_size = ring->size * ring->stride; in mlx4_en_activate_rx_rings()
453 memset(ring->buf, 0, ring->buf_size); in mlx4_en_activate_rx_rings()
454 mlx4_en_update_rx_prod_db(ring); in mlx4_en_activate_rx_rings()
457 for (i = 0; i < ring->size; i++) in mlx4_en_activate_rx_rings()
458 mlx4_en_init_rx_desc(priv, ring, i); in mlx4_en_activate_rx_rings()
461 err = mlx4_en_init_allocator(priv, ring); in mlx4_en_activate_rx_rings()
464 if (ring->stride <= TXBB_SIZE) in mlx4_en_activate_rx_rings()
465 ring->buf -= TXBB_SIZE; in mlx4_en_activate_rx_rings()
475 ring = priv->rx_ring[ring_ind]; in mlx4_en_activate_rx_rings()
477 ring->size_mask = ring->actual_size - 1; in mlx4_en_activate_rx_rings()
478 mlx4_en_update_rx_prod_db(ring); in mlx4_en_activate_rx_rings()
504 int ring; in mlx4_en_recover_from_oom() local
509 for (ring = 0; ring < priv->rx_ring_num; ring++) { in mlx4_en_recover_from_oom()
510 if (mlx4_en_is_ring_empty(priv->rx_ring[ring])) { in mlx4_en_recover_from_oom()
512 napi_reschedule(&priv->rx_cq[ring]->napi); in mlx4_en_recover_from_oom()
523 struct mlx4_en_rx_ring *ring = *pring; in mlx4_en_destroy_rx_ring() local
525 mlx4_en_unmap_buffer(&ring->wqres.buf); in mlx4_en_destroy_rx_ring()
526 mlx4_free_hwq_res(mdev->dev, &ring->wqres, size * stride + TXBB_SIZE); in mlx4_en_destroy_rx_ring()
527 vfree(ring->rx_info); in mlx4_en_destroy_rx_ring()
528 ring->rx_info = NULL; in mlx4_en_destroy_rx_ring()
529 kfree(ring); in mlx4_en_destroy_rx_ring()
537 struct mlx4_en_rx_ring *ring) in mlx4_en_deactivate_rx_ring() argument
539 mlx4_en_free_rx_buf(priv, ring); in mlx4_en_deactivate_rx_ring()
540 if (ring->stride <= TXBB_SIZE) in mlx4_en_deactivate_rx_ring()
541 ring->buf -= TXBB_SIZE; in mlx4_en_deactivate_rx_ring()
542 mlx4_en_destroy_allocator(priv, ring); in mlx4_en_deactivate_rx_ring()
665 struct mlx4_en_rx_ring *ring) in mlx4_en_refill_rx_buffers() argument
667 int index = ring->prod & ring->size_mask; in mlx4_en_refill_rx_buffers()
669 while ((u32) (ring->prod - ring->cons) < ring->actual_size) { in mlx4_en_refill_rx_buffers()
670 if (mlx4_en_prepare_rx_desc(priv, ring, index, GFP_ATOMIC)) in mlx4_en_refill_rx_buffers()
672 ring->prod++; in mlx4_en_refill_rx_buffers()
673 index = ring->prod & ring->size_mask; in mlx4_en_refill_rx_buffers()
682 struct mlx4_en_rx_ring *ring = priv->rx_ring[cq->ring]; in mlx4_en_process_rx_cq() local
704 index = cq->mcq.cons_index & ring->size_mask; in mlx4_en_process_rx_cq()
711 frags = ring->rx_info + (index << priv->log_rx_info); in mlx4_en_process_rx_cq()
712 rx_desc = ring->buf + (index << ring->log_stride); in mlx4_en_process_rx_cq()
771 length -= ring->fcs_del; in mlx4_en_process_rx_cq()
772 ring->bytes += length; in mlx4_en_process_rx_cq()
773 ring->packets++; in mlx4_en_process_rx_cq()
780 ring->csum_ok++; in mlx4_en_process_rx_cq()
820 skb_record_rx_queue(gro_skb, cq->ring); in mlx4_en_process_rx_cq()
823 if (ring->hwtstamp_rx_filter == HWTSTAMP_FILTER_ALL) { in mlx4_en_process_rx_cq()
838 ring->csum_none++; in mlx4_en_process_rx_cq()
842 ring->csum_none++; in mlx4_en_process_rx_cq()
858 skb_record_rx_queue(skb, cq->ring); in mlx4_en_process_rx_cq()
873 if (ring->hwtstamp_rx_filter == HWTSTAMP_FILTER_ALL) { in mlx4_en_process_rx_cq()
891 index = (cq->mcq.cons_index) & ring->size_mask; in mlx4_en_process_rx_cq()
901 ring->cons = cq->mcq.cons_index; in mlx4_en_process_rx_cq()
902 mlx4_en_refill_rx_buffers(priv, ring); in mlx4_en_process_rx_cq()
903 mlx4_en_update_rx_prod_db(ring); in mlx4_en_process_rx_cq()
1013 struct mlx4_en_rx_ring *ring, in mlx4_en_config_rss_qp() argument
1033 mlx4_en_fill_qp_context(priv, ring->actual_size, ring->stride, 0, 0, in mlx4_en_config_rss_qp()
1034 qpn, ring->cqn, -1, context); in mlx4_en_config_rss_qp()
1035 context->db_rec_addr = cpu_to_be64(ring->wqres.db.dma); in mlx4_en_config_rss_qp()
1040 ring->fcs_del = ETH_FCS_LEN; in mlx4_en_config_rss_qp()
1042 ring->fcs_del = 0; in mlx4_en_config_rss_qp()
1044 err = mlx4_qp_to_ready(mdev->dev, &ring->wqres.mtt, context, qp, state); in mlx4_en_config_rss_qp()
1049 mlx4_en_update_rx_prod_db(ring); in mlx4_en_config_rss_qp()