Lines Matching refs:rxq
129 #define MVNETA_CPU_RXQ_ACCESS(rxq) BIT(rxq) argument
817 struct mvneta_rx_queue *rxq, in mvneta_rxq_non_occup_desc_add() argument
824 mvreg_write(pp, MVNETA_RXQ_STATUS_UPDATE_REG(rxq->id), in mvneta_rxq_non_occup_desc_add()
830 mvreg_write(pp, MVNETA_RXQ_STATUS_UPDATE_REG(rxq->id), in mvneta_rxq_non_occup_desc_add()
836 struct mvneta_rx_queue *rxq) in mvneta_rxq_busy_desc_num_get() argument
840 val = mvreg_read(pp, MVNETA_RXQ_STATUS_REG(rxq->id)); in mvneta_rxq_busy_desc_num_get()
848 struct mvneta_rx_queue *rxq, in mvneta_rxq_desc_num_update() argument
856 mvreg_write(pp, MVNETA_RXQ_STATUS_UPDATE_REG(rxq->id), val); in mvneta_rxq_desc_num_update()
876 mvreg_write(pp, MVNETA_RXQ_STATUS_UPDATE_REG(rxq->id), val); in mvneta_rxq_desc_num_update()
882 mvneta_rxq_next_desc_get(struct mvneta_rx_queue *rxq) in mvneta_rxq_next_desc_get() argument
884 int rx_desc = rxq->next_desc_to_proc; in mvneta_rxq_next_desc_get()
886 rxq->next_desc_to_proc = MVNETA_QUEUE_NEXT_DESC(rxq, rx_desc); in mvneta_rxq_next_desc_get()
887 prefetch(rxq->descs + rxq->next_desc_to_proc); in mvneta_rxq_next_desc_get()
888 return rxq->descs + rx_desc; in mvneta_rxq_next_desc_get()
906 struct mvneta_rx_queue *rxq, in mvneta_rxq_offset_set() argument
911 val = mvreg_read(pp, MVNETA_RXQ_CONFIG_REG(rxq->id)); in mvneta_rxq_offset_set()
916 mvreg_write(pp, MVNETA_RXQ_CONFIG_REG(rxq->id), val); in mvneta_rxq_offset_set()
963 struct mvneta_rx_queue *rxq, in mvneta_rxq_buf_size_set() argument
968 val = mvreg_read(pp, MVNETA_RXQ_SIZE_REG(rxq->id)); in mvneta_rxq_buf_size_set()
973 mvreg_write(pp, MVNETA_RXQ_SIZE_REG(rxq->id), val); in mvneta_rxq_buf_size_set()
978 struct mvneta_rx_queue *rxq) in mvneta_rxq_bm_disable() argument
982 val = mvreg_read(pp, MVNETA_RXQ_CONFIG_REG(rxq->id)); in mvneta_rxq_bm_disable()
984 mvreg_write(pp, MVNETA_RXQ_CONFIG_REG(rxq->id), val); in mvneta_rxq_bm_disable()
989 struct mvneta_rx_queue *rxq) in mvneta_rxq_bm_enable() argument
993 val = mvreg_read(pp, MVNETA_RXQ_CONFIG_REG(rxq->id)); in mvneta_rxq_bm_enable()
995 mvreg_write(pp, MVNETA_RXQ_CONFIG_REG(rxq->id), val); in mvneta_rxq_bm_enable()
1000 struct mvneta_rx_queue *rxq) in mvneta_rxq_long_pool_set() argument
1004 val = mvreg_read(pp, MVNETA_RXQ_CONFIG_REG(rxq->id)); in mvneta_rxq_long_pool_set()
1008 mvreg_write(pp, MVNETA_RXQ_CONFIG_REG(rxq->id), val); in mvneta_rxq_long_pool_set()
1013 struct mvneta_rx_queue *rxq) in mvneta_rxq_short_pool_set() argument
1017 val = mvreg_read(pp, MVNETA_RXQ_CONFIG_REG(rxq->id)); in mvneta_rxq_short_pool_set()
1021 mvreg_write(pp, MVNETA_RXQ_CONFIG_REG(rxq->id), val); in mvneta_rxq_short_pool_set()
1229 struct mvneta_rx_queue *rxq = &pp->rxqs[queue]; in mvneta_port_up() local
1231 if (rxq->descs) in mvneta_port_up()
1458 int rxq, txq; in mvneta_defaults_set() local
1460 for (rxq = 0; rxq < rxq_number; rxq++) in mvneta_defaults_set()
1461 if ((rxq % max_cpu) == cpu) in mvneta_defaults_set()
1462 rxq_map |= MVNETA_CPU_RXQ_ACCESS(rxq); in mvneta_defaults_set()
1646 struct mvneta_rx_queue *rxq, u32 value) in mvneta_rx_pkts_coal_set() argument
1648 mvreg_write(pp, MVNETA_RXQ_THRESHOLD_REG(rxq->id), in mvneta_rx_pkts_coal_set()
1656 struct mvneta_rx_queue *rxq, u32 value) in mvneta_rx_time_coal_set() argument
1664 mvreg_write(pp, MVNETA_RXQ_TIME_COAL_REG(rxq->id), val); in mvneta_rx_time_coal_set()
1684 struct mvneta_rx_queue *rxq) in mvneta_rx_desc_fill() argument
1689 i = rx_desc - rxq->descs; in mvneta_rx_desc_fill()
1690 rxq->buf_virt_addr[i] = virt_addr; in mvneta_rx_desc_fill()
1892 struct mvneta_rx_queue *rxq, in mvneta_rx_refill() argument
1898 page = page_pool_alloc_pages(rxq->page_pool, in mvneta_rx_refill()
1904 mvneta_rx_desc_fill(rx_desc, phys_addr, page, rxq); in mvneta_rx_refill()
1942 struct mvneta_rx_queue *rxq) in mvneta_rxq_drop_pkts() argument
1946 rx_done = mvneta_rxq_busy_desc_num_get(pp, rxq); in mvneta_rxq_drop_pkts()
1948 mvneta_rxq_desc_num_update(pp, rxq, rx_done, rx_done); in mvneta_rxq_drop_pkts()
1953 mvneta_rxq_next_desc_get(rxq); in mvneta_rxq_drop_pkts()
1965 for (i = 0; i < rxq->size; i++) { in mvneta_rxq_drop_pkts()
1966 struct mvneta_rx_desc *rx_desc = rxq->descs + i; in mvneta_rxq_drop_pkts()
1967 void *data = rxq->buf_virt_addr[i]; in mvneta_rxq_drop_pkts()
1971 page_pool_put_full_page(rxq->page_pool, data, false); in mvneta_rxq_drop_pkts()
1973 if (xdp_rxq_info_is_reg(&rxq->xdp_rxq)) in mvneta_rxq_drop_pkts()
1974 xdp_rxq_info_unreg(&rxq->xdp_rxq); in mvneta_rxq_drop_pkts()
1975 page_pool_destroy(rxq->page_pool); in mvneta_rxq_drop_pkts()
1976 rxq->page_pool = NULL; in mvneta_rxq_drop_pkts()
1996 int mvneta_rx_refill_queue(struct mvneta_port *pp, struct mvneta_rx_queue *rxq) in mvneta_rx_refill_queue() argument
1999 int curr_desc = rxq->first_to_refill; in mvneta_rx_refill_queue()
2002 for (i = 0; (i < rxq->refill_num) && (i < 64); i++) { in mvneta_rx_refill_queue()
2003 rx_desc = rxq->descs + curr_desc; in mvneta_rx_refill_queue()
2005 if (mvneta_rx_refill(pp, rx_desc, rxq, GFP_ATOMIC)) { in mvneta_rx_refill_queue()
2009 rxq->id, i, rxq->refill_num); in mvneta_rx_refill_queue()
2018 curr_desc = MVNETA_QUEUE_NEXT_DESC(rxq, curr_desc); in mvneta_rx_refill_queue()
2020 rxq->refill_num -= i; in mvneta_rx_refill_queue()
2021 rxq->first_to_refill = curr_desc; in mvneta_rx_refill_queue()
2027 mvneta_xdp_put_buff(struct mvneta_port *pp, struct mvneta_rx_queue *rxq, in mvneta_xdp_put_buff() argument
2034 page_pool_put_full_page(rxq->page_pool, in mvneta_xdp_put_buff()
2036 page_pool_put_page(rxq->page_pool, virt_to_head_page(xdp->data), in mvneta_xdp_put_buff()
2170 mvneta_run_xdp(struct mvneta_port *pp, struct mvneta_rx_queue *rxq, in mvneta_run_xdp() argument
2194 mvneta_xdp_put_buff(pp, rxq, xdp, sync, true); in mvneta_run_xdp()
2205 mvneta_xdp_put_buff(pp, rxq, xdp, sync, true); in mvneta_run_xdp()
2214 mvneta_xdp_put_buff(pp, rxq, xdp, sync, true); in mvneta_run_xdp()
2229 struct mvneta_rx_queue *rxq, in mvneta_swbm_rx_frame() argument
2248 dma_dir = page_pool_get_dma_dir(rxq->page_pool); in mvneta_swbm_rx_frame()
2270 struct mvneta_rx_queue *rxq, in mvneta_swbm_add_rx_fragment() argument
2286 dma_dir = page_pool_get_dma_dir(rxq->page_pool); in mvneta_swbm_add_rx_fragment()
2300 page_pool_put_full_page(rxq->page_pool, page, true); in mvneta_swbm_add_rx_fragment()
2306 mvneta_swbm_build_skb(struct mvneta_port *pp, struct mvneta_rx_queue *rxq, in mvneta_swbm_build_skb() argument
2317 page_pool_release_page(rxq->page_pool, virt_to_page(xdp->data)); in mvneta_swbm_build_skb()
2329 page_pool_release_page(rxq->page_pool, skb_frag_page(frag)); in mvneta_swbm_build_skb()
2338 struct mvneta_rx_queue *rxq) in mvneta_rx_swbm() argument
2344 .rxq = &rxq->xdp_rxq, in mvneta_rx_swbm()
2351 rx_todo = mvneta_rxq_busy_desc_num_get(pp, rxq); in mvneta_rx_swbm()
2358 struct mvneta_rx_desc *rx_desc = mvneta_rxq_next_desc_get(rxq); in mvneta_rx_swbm()
2363 index = rx_desc - rxq->descs; in mvneta_rx_swbm()
2364 page = (struct page *)rxq->buf_virt_addr[index]; in mvneta_rx_swbm()
2368 rxq->refill_num++; in mvneta_rx_swbm()
2381 mvneta_swbm_rx_frame(pp, rx_desc, rxq, &xdp_buf, in mvneta_rx_swbm()
2386 page_pool_put_full_page(rxq->page_pool, page, in mvneta_rx_swbm()
2391 mvneta_swbm_add_rx_fragment(pp, rx_desc, rxq, &xdp_buf, in mvneta_rx_swbm()
2400 mvneta_xdp_put_buff(pp, rxq, &xdp_buf, -1, true); in mvneta_rx_swbm()
2405 mvneta_run_xdp(pp, rxq, xdp_prog, &xdp_buf, frame_sz, &ps)) in mvneta_rx_swbm()
2408 skb = mvneta_swbm_build_skb(pp, rxq, &xdp_buf, desc_status); in mvneta_rx_swbm()
2412 mvneta_xdp_put_buff(pp, rxq, &xdp_buf, -1, true); in mvneta_rx_swbm()
2433 mvneta_xdp_put_buff(pp, rxq, &xdp_buf, -1, true); in mvneta_rx_swbm()
2442 refill = mvneta_rx_refill_queue(pp, rxq); in mvneta_rx_swbm()
2445 mvneta_rxq_desc_num_update(pp, rxq, rx_proc, refill); in mvneta_rx_swbm()
2453 struct mvneta_rx_queue *rxq) in mvneta_rx_hwbm() argument
2461 rx_done = mvneta_rxq_busy_desc_num_get(pp, rxq); in mvneta_rx_hwbm()
2470 struct mvneta_rx_desc *rx_desc = mvneta_rxq_next_desc_get(rxq); in mvneta_rx_hwbm()
2579 mvneta_rxq_desc_num_update(pp, rxq, rx_done, rx_done); in mvneta_rx_hwbm()
3201 struct mvneta_rx_queue *rxq, int size) in mvneta_create_page_pool() argument
3216 rxq->page_pool = page_pool_create(&pp_params); in mvneta_create_page_pool()
3217 if (IS_ERR(rxq->page_pool)) { in mvneta_create_page_pool()
3218 err = PTR_ERR(rxq->page_pool); in mvneta_create_page_pool()
3219 rxq->page_pool = NULL; in mvneta_create_page_pool()
3223 err = xdp_rxq_info_reg(&rxq->xdp_rxq, pp->dev, rxq->id); in mvneta_create_page_pool()
3227 err = xdp_rxq_info_reg_mem_model(&rxq->xdp_rxq, MEM_TYPE_PAGE_POOL, in mvneta_create_page_pool()
3228 rxq->page_pool); in mvneta_create_page_pool()
3235 xdp_rxq_info_unreg(&rxq->xdp_rxq); in mvneta_create_page_pool()
3237 page_pool_destroy(rxq->page_pool); in mvneta_create_page_pool()
3238 rxq->page_pool = NULL; in mvneta_create_page_pool()
3243 static int mvneta_rxq_fill(struct mvneta_port *pp, struct mvneta_rx_queue *rxq, in mvneta_rxq_fill() argument
3248 err = mvneta_create_page_pool(pp, rxq, num); in mvneta_rxq_fill()
3253 memset(rxq->descs + i, 0, sizeof(struct mvneta_rx_desc)); in mvneta_rxq_fill()
3254 if (mvneta_rx_refill(pp, rxq->descs + i, rxq, in mvneta_rxq_fill()
3258 __func__, rxq->id, i, num); in mvneta_rxq_fill()
3266 mvneta_rxq_non_occup_desc_add(pp, rxq, i); in mvneta_rxq_fill()
3293 struct mvneta_rx_queue *rxq) in mvneta_rxq_sw_init() argument
3295 rxq->size = pp->rx_ring_size; in mvneta_rxq_sw_init()
3298 rxq->descs = dma_alloc_coherent(pp->dev->dev.parent, in mvneta_rxq_sw_init()
3299 rxq->size * MVNETA_DESC_ALIGNED_SIZE, in mvneta_rxq_sw_init()
3300 &rxq->descs_phys, GFP_KERNEL); in mvneta_rxq_sw_init()
3301 if (!rxq->descs) in mvneta_rxq_sw_init()
3304 rxq->last_desc = rxq->size - 1; in mvneta_rxq_sw_init()
3310 struct mvneta_rx_queue *rxq) in mvneta_rxq_hw_init() argument
3313 mvreg_write(pp, MVNETA_RXQ_BASE_ADDR_REG(rxq->id), rxq->descs_phys); in mvneta_rxq_hw_init()
3314 mvreg_write(pp, MVNETA_RXQ_SIZE_REG(rxq->id), rxq->size); in mvneta_rxq_hw_init()
3317 mvneta_rx_pkts_coal_set(pp, rxq, rxq->pkts_coal); in mvneta_rxq_hw_init()
3318 mvneta_rx_time_coal_set(pp, rxq, rxq->time_coal); in mvneta_rxq_hw_init()
3322 mvneta_rxq_offset_set(pp, rxq, 0); in mvneta_rxq_hw_init()
3323 mvneta_rxq_buf_size_set(pp, rxq, PAGE_SIZE < SZ_64K ? in mvneta_rxq_hw_init()
3326 mvneta_rxq_bm_disable(pp, rxq); in mvneta_rxq_hw_init()
3327 mvneta_rxq_fill(pp, rxq, rxq->size); in mvneta_rxq_hw_init()
3330 mvneta_rxq_offset_set(pp, rxq, in mvneta_rxq_hw_init()
3333 mvneta_rxq_bm_enable(pp, rxq); in mvneta_rxq_hw_init()
3335 mvneta_rxq_long_pool_set(pp, rxq); in mvneta_rxq_hw_init()
3336 mvneta_rxq_short_pool_set(pp, rxq); in mvneta_rxq_hw_init()
3337 mvneta_rxq_non_occup_desc_add(pp, rxq, rxq->size); in mvneta_rxq_hw_init()
3343 struct mvneta_rx_queue *rxq) in mvneta_rxq_init() argument
3348 ret = mvneta_rxq_sw_init(pp, rxq); in mvneta_rxq_init()
3352 mvneta_rxq_hw_init(pp, rxq); in mvneta_rxq_init()
3359 struct mvneta_rx_queue *rxq) in mvneta_rxq_deinit() argument
3361 mvneta_rxq_drop_pkts(pp, rxq); in mvneta_rxq_deinit()
3363 if (rxq->descs) in mvneta_rxq_deinit()
3365 rxq->size * MVNETA_DESC_ALIGNED_SIZE, in mvneta_rxq_deinit()
3366 rxq->descs, in mvneta_rxq_deinit()
3367 rxq->descs_phys); in mvneta_rxq_deinit()
3369 rxq->descs = NULL; in mvneta_rxq_deinit()
3370 rxq->last_desc = 0; in mvneta_rxq_deinit()
3371 rxq->next_desc_to_proc = 0; in mvneta_rxq_deinit()
3372 rxq->descs_phys = 0; in mvneta_rxq_deinit()
3373 rxq->first_to_refill = 0; in mvneta_rxq_deinit()
3374 rxq->refill_num = 0; in mvneta_rxq_deinit()
4150 int rxq; in mvneta_percpu_elect() local
4152 for (rxq = 0; rxq < rxq_number; rxq++) in mvneta_percpu_elect()
4153 if ((rxq % max_cpu) == cpu) in mvneta_percpu_elect()
4154 rxq_map |= MVNETA_CPU_RXQ_ACCESS(rxq); in mvneta_percpu_elect()
4490 struct mvneta_rx_queue *rxq = &pp->rxqs[queue]; in mvneta_ethtool_set_coalesce() local
4491 rxq->time_coal = c->rx_coalesce_usecs; in mvneta_ethtool_set_coalesce()
4492 rxq->pkts_coal = c->rx_max_coalesced_frames; in mvneta_ethtool_set_coalesce()
4493 mvneta_rx_pkts_coal_set(pp, rxq, rxq->pkts_coal); in mvneta_ethtool_set_coalesce()
4494 mvneta_rx_time_coal_set(pp, rxq, rxq->time_coal); in mvneta_ethtool_set_coalesce()
4976 struct mvneta_rx_queue *rxq = &pp->rxqs[queue]; in mvneta_init() local
4977 rxq->id = queue; in mvneta_init()
4978 rxq->size = pp->rx_ring_size; in mvneta_init()
4979 rxq->pkts_coal = MVNETA_RX_COAL_PKTS; in mvneta_init()
4980 rxq->time_coal = MVNETA_RX_COAL_USEC; in mvneta_init()
4981 rxq->buf_virt_addr in mvneta_init()
4983 rxq->size, in mvneta_init()
4984 sizeof(*rxq->buf_virt_addr), in mvneta_init()
4986 if (!rxq->buf_virt_addr) in mvneta_init()
5357 struct mvneta_rx_queue *rxq = &pp->rxqs[queue]; in mvneta_suspend() local
5359 mvneta_rxq_drop_pkts(pp, rxq); in mvneta_suspend()
5409 struct mvneta_rx_queue *rxq = &pp->rxqs[queue]; in mvneta_resume() local
5411 rxq->next_desc_to_proc = 0; in mvneta_resume()
5412 mvneta_rxq_hw_init(pp, rxq); in mvneta_resume()