Lines Matching refs:priv
133 static void stmmac_tx_timer_arm(struct stmmac_priv *priv, u32 queue);
134 static void stmmac_flush_tx_descriptors(struct stmmac_priv *priv, int queue);
144 int stmmac_bus_clks_config(struct stmmac_priv *priv, bool enabled) in stmmac_bus_clks_config() argument
149 ret = clk_prepare_enable(priv->plat->stmmac_clk); in stmmac_bus_clks_config()
152 ret = clk_prepare_enable(priv->plat->pclk); in stmmac_bus_clks_config()
154 clk_disable_unprepare(priv->plat->stmmac_clk); in stmmac_bus_clks_config()
157 if (priv->plat->clks_config) { in stmmac_bus_clks_config()
158 ret = priv->plat->clks_config(priv->plat->bsp_priv, enabled); in stmmac_bus_clks_config()
160 clk_disable_unprepare(priv->plat->stmmac_clk); in stmmac_bus_clks_config()
161 clk_disable_unprepare(priv->plat->pclk); in stmmac_bus_clks_config()
166 clk_disable_unprepare(priv->plat->stmmac_clk); in stmmac_bus_clks_config()
167 clk_disable_unprepare(priv->plat->pclk); in stmmac_bus_clks_config()
168 if (priv->plat->clks_config) in stmmac_bus_clks_config()
169 priv->plat->clks_config(priv->plat->bsp_priv, enabled); in stmmac_bus_clks_config()
197 static void __stmmac_disable_all_queues(struct stmmac_priv *priv) in __stmmac_disable_all_queues() argument
199 u32 rx_queues_cnt = priv->plat->rx_queues_to_use; in __stmmac_disable_all_queues()
200 u32 tx_queues_cnt = priv->plat->tx_queues_to_use; in __stmmac_disable_all_queues()
205 struct stmmac_channel *ch = &priv->channel[queue]; in __stmmac_disable_all_queues()
207 if (stmmac_xdp_is_enabled(priv) && in __stmmac_disable_all_queues()
208 test_bit(queue, priv->af_xdp_zc_qps)) { in __stmmac_disable_all_queues()
224 static void stmmac_disable_all_queues(struct stmmac_priv *priv) in stmmac_disable_all_queues() argument
226 u32 rx_queues_cnt = priv->plat->rx_queues_to_use; in stmmac_disable_all_queues()
232 rx_q = &priv->rx_queue[queue]; in stmmac_disable_all_queues()
239 __stmmac_disable_all_queues(priv); in stmmac_disable_all_queues()
246 static void stmmac_enable_all_queues(struct stmmac_priv *priv) in stmmac_enable_all_queues() argument
248 u32 rx_queues_cnt = priv->plat->rx_queues_to_use; in stmmac_enable_all_queues()
249 u32 tx_queues_cnt = priv->plat->tx_queues_to_use; in stmmac_enable_all_queues()
254 struct stmmac_channel *ch = &priv->channel[queue]; in stmmac_enable_all_queues()
256 if (stmmac_xdp_is_enabled(priv) && in stmmac_enable_all_queues()
257 test_bit(queue, priv->af_xdp_zc_qps)) { in stmmac_enable_all_queues()
269 static void stmmac_service_event_schedule(struct stmmac_priv *priv) in stmmac_service_event_schedule() argument
271 if (!test_bit(STMMAC_DOWN, &priv->state) && in stmmac_service_event_schedule()
272 !test_and_set_bit(STMMAC_SERVICE_SCHED, &priv->state)) in stmmac_service_event_schedule()
273 queue_work(priv->wq, &priv->service_task); in stmmac_service_event_schedule()
276 static void stmmac_global_err(struct stmmac_priv *priv) in stmmac_global_err() argument
278 netif_carrier_off(priv->dev); in stmmac_global_err()
279 set_bit(STMMAC_RESET_REQUESTED, &priv->state); in stmmac_global_err()
280 stmmac_service_event_schedule(priv); in stmmac_global_err()
295 static void stmmac_clk_csr_set(struct stmmac_priv *priv) in stmmac_clk_csr_set() argument
299 clk_rate = clk_get_rate(priv->plat->stmmac_clk); in stmmac_clk_csr_set()
308 if (!(priv->clk_csr & MAC_CSR_H_FRQ_MASK)) { in stmmac_clk_csr_set()
310 priv->clk_csr = STMMAC_CSR_20_35M; in stmmac_clk_csr_set()
312 priv->clk_csr = STMMAC_CSR_35_60M; in stmmac_clk_csr_set()
314 priv->clk_csr = STMMAC_CSR_60_100M; in stmmac_clk_csr_set()
316 priv->clk_csr = STMMAC_CSR_100_150M; in stmmac_clk_csr_set()
318 priv->clk_csr = STMMAC_CSR_150_250M; in stmmac_clk_csr_set()
320 priv->clk_csr = STMMAC_CSR_250_300M; in stmmac_clk_csr_set()
323 if (priv->plat->has_sun8i) { in stmmac_clk_csr_set()
325 priv->clk_csr = 0x03; in stmmac_clk_csr_set()
327 priv->clk_csr = 0x02; in stmmac_clk_csr_set()
329 priv->clk_csr = 0x01; in stmmac_clk_csr_set()
331 priv->clk_csr = 0; in stmmac_clk_csr_set()
334 if (priv->plat->has_xgmac) { in stmmac_clk_csr_set()
336 priv->clk_csr = 0x5; in stmmac_clk_csr_set()
338 priv->clk_csr = 0x4; in stmmac_clk_csr_set()
340 priv->clk_csr = 0x3; in stmmac_clk_csr_set()
342 priv->clk_csr = 0x2; in stmmac_clk_csr_set()
344 priv->clk_csr = 0x1; in stmmac_clk_csr_set()
346 priv->clk_csr = 0x0; in stmmac_clk_csr_set()
356 static inline u32 stmmac_tx_avail(struct stmmac_priv *priv, u32 queue) in stmmac_tx_avail() argument
358 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; in stmmac_tx_avail()
364 avail = priv->dma_tx_size - tx_q->cur_tx + tx_q->dirty_tx - 1; in stmmac_tx_avail()
374 static inline u32 stmmac_rx_dirty(struct stmmac_priv *priv, u32 queue) in stmmac_rx_dirty() argument
376 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; in stmmac_rx_dirty()
382 dirty = priv->dma_rx_size - rx_q->dirty_rx + rx_q->cur_rx; in stmmac_rx_dirty()
387 static void stmmac_lpi_entry_timer_config(struct stmmac_priv *priv, bool en) in stmmac_lpi_entry_timer_config() argument
392 priv->eee_sw_timer_en = en ? 0 : 1; in stmmac_lpi_entry_timer_config()
393 tx_lpi_timer = en ? priv->tx_lpi_timer : 0; in stmmac_lpi_entry_timer_config()
394 stmmac_set_eee_lpi_timer(priv, priv->hw, tx_lpi_timer); in stmmac_lpi_entry_timer_config()
403 static int stmmac_enable_eee_mode(struct stmmac_priv *priv) in stmmac_enable_eee_mode() argument
405 u32 tx_cnt = priv->plat->tx_queues_to_use; in stmmac_enable_eee_mode()
410 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; in stmmac_enable_eee_mode()
417 if (!priv->tx_path_in_lpi_mode) in stmmac_enable_eee_mode()
418 stmmac_set_eee_mode(priv, priv->hw, in stmmac_enable_eee_mode()
419 priv->plat->en_tx_lpi_clockgating); in stmmac_enable_eee_mode()
429 void stmmac_disable_eee_mode(struct stmmac_priv *priv) in stmmac_disable_eee_mode() argument
431 if (!priv->eee_sw_timer_en) { in stmmac_disable_eee_mode()
432 stmmac_lpi_entry_timer_config(priv, 0); in stmmac_disable_eee_mode()
436 stmmac_reset_eee_mode(priv, priv->hw); in stmmac_disable_eee_mode()
437 del_timer_sync(&priv->eee_ctrl_timer); in stmmac_disable_eee_mode()
438 priv->tx_path_in_lpi_mode = false; in stmmac_disable_eee_mode()
450 struct stmmac_priv *priv = from_timer(priv, t, eee_ctrl_timer); in stmmac_eee_ctrl_timer() local
452 if (stmmac_enable_eee_mode(priv)) in stmmac_eee_ctrl_timer()
453 mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(priv->tx_lpi_timer)); in stmmac_eee_ctrl_timer()
464 bool stmmac_eee_init(struct stmmac_priv *priv) in stmmac_eee_init() argument
466 int eee_tw_timer = priv->eee_tw_timer; in stmmac_eee_init()
471 if (priv->hw->pcs == STMMAC_PCS_TBI || in stmmac_eee_init()
472 priv->hw->pcs == STMMAC_PCS_RTBI) in stmmac_eee_init()
476 if (!priv->dma_cap.eee) in stmmac_eee_init()
479 mutex_lock(&priv->lock); in stmmac_eee_init()
482 if (!priv->eee_active) { in stmmac_eee_init()
483 if (priv->eee_enabled) { in stmmac_eee_init()
484 netdev_dbg(priv->dev, "disable EEE\n"); in stmmac_eee_init()
485 stmmac_lpi_entry_timer_config(priv, 0); in stmmac_eee_init()
486 del_timer_sync(&priv->eee_ctrl_timer); in stmmac_eee_init()
487 stmmac_set_eee_timer(priv, priv->hw, 0, eee_tw_timer); in stmmac_eee_init()
488 if (priv->hw->xpcs) in stmmac_eee_init()
489 xpcs_config_eee(priv->hw->xpcs, in stmmac_eee_init()
490 priv->plat->mult_fact_100ns, in stmmac_eee_init()
493 mutex_unlock(&priv->lock); in stmmac_eee_init()
497 if (priv->eee_active && !priv->eee_enabled) { in stmmac_eee_init()
498 timer_setup(&priv->eee_ctrl_timer, stmmac_eee_ctrl_timer, 0); in stmmac_eee_init()
499 stmmac_set_eee_timer(priv, priv->hw, STMMAC_DEFAULT_LIT_LS, in stmmac_eee_init()
501 if (priv->hw->xpcs) in stmmac_eee_init()
502 xpcs_config_eee(priv->hw->xpcs, in stmmac_eee_init()
503 priv->plat->mult_fact_100ns, in stmmac_eee_init()
507 if (priv->plat->has_gmac4 && priv->tx_lpi_timer <= STMMAC_ET_MAX) { in stmmac_eee_init()
508 del_timer_sync(&priv->eee_ctrl_timer); in stmmac_eee_init()
509 priv->tx_path_in_lpi_mode = false; in stmmac_eee_init()
510 stmmac_lpi_entry_timer_config(priv, 1); in stmmac_eee_init()
512 stmmac_lpi_entry_timer_config(priv, 0); in stmmac_eee_init()
513 mod_timer(&priv->eee_ctrl_timer, in stmmac_eee_init()
514 STMMAC_LPI_T(priv->tx_lpi_timer)); in stmmac_eee_init()
517 mutex_unlock(&priv->lock); in stmmac_eee_init()
518 netdev_dbg(priv->dev, "Energy-Efficient Ethernet initialized\n"); in stmmac_eee_init()
522 static inline u32 stmmac_cdc_adjust(struct stmmac_priv *priv) in stmmac_cdc_adjust() argument
525 if (priv->plat->has_gmac4 && priv->plat->clk_ptp_rate) in stmmac_cdc_adjust()
526 return (2 * NSEC_PER_SEC) / priv->plat->clk_ptp_rate; in stmmac_cdc_adjust()
538 static void stmmac_get_tx_hwtstamp(struct stmmac_priv *priv, in stmmac_get_tx_hwtstamp() argument
545 if (!priv->hwts_tx_en) in stmmac_get_tx_hwtstamp()
553 if (stmmac_get_tx_timestamp_status(priv, p)) { in stmmac_get_tx_hwtstamp()
554 stmmac_get_timestamp(priv, p, priv->adv_ts, &ns); in stmmac_get_tx_hwtstamp()
556 } else if (!stmmac_get_mac_tx_timestamp(priv, priv->hw, &ns)) { in stmmac_get_tx_hwtstamp()
561 ns -= stmmac_cdc_adjust(priv); in stmmac_get_tx_hwtstamp()
566 netdev_dbg(priv->dev, "get valid TX hw timestamp %llu\n", ns); in stmmac_get_tx_hwtstamp()
581 static void stmmac_get_rx_hwtstamp(struct stmmac_priv *priv, struct dma_desc *p, in stmmac_get_rx_hwtstamp() argument
588 if (!priv->hwts_rx_en) in stmmac_get_rx_hwtstamp()
591 if (priv->plat->has_gmac4 || priv->plat->has_xgmac) in stmmac_get_rx_hwtstamp()
595 if (stmmac_get_rx_timestamp_status(priv, p, np, priv->adv_ts)) { in stmmac_get_rx_hwtstamp()
596 stmmac_get_timestamp(priv, desc, priv->adv_ts, &ns); in stmmac_get_rx_hwtstamp()
598 ns -= stmmac_cdc_adjust(priv); in stmmac_get_rx_hwtstamp()
600 netdev_dbg(priv->dev, "get valid RX hw timestamp %llu\n", ns); in stmmac_get_rx_hwtstamp()
605 netdev_dbg(priv->dev, "cannot get RX hw timestamp\n"); in stmmac_get_rx_hwtstamp()
622 struct stmmac_priv *priv = netdev_priv(dev); in stmmac_hwtstamp_set() local
633 if (!(priv->dma_cap.time_stamp || priv->adv_ts)) { in stmmac_hwtstamp_set()
634 netdev_alert(priv->dev, "No support for HW time stamping\n"); in stmmac_hwtstamp_set()
635 priv->hwts_tx_en = 0; in stmmac_hwtstamp_set()
636 priv->hwts_rx_en = 0; in stmmac_hwtstamp_set()
645 netdev_dbg(priv->dev, "%s config flags:0x%x, tx_type:0x%x, rx_filter:0x%x\n", in stmmac_hwtstamp_set()
656 if (priv->adv_ts) { in stmmac_hwtstamp_set()
737 if (priv->synopsys_id < DWMAC_CORE_4_10) in stmmac_hwtstamp_set()
790 priv->hwts_rx_en = ((config.rx_filter == HWTSTAMP_FILTER_NONE) ? 0 : 1); in stmmac_hwtstamp_set()
791 priv->hwts_tx_en = config.tx_type == HWTSTAMP_TX_ON; in stmmac_hwtstamp_set()
793 priv->systime_flags = STMMAC_HWTS_ACTIVE; in stmmac_hwtstamp_set()
795 if (priv->hwts_tx_en || priv->hwts_rx_en) { in stmmac_hwtstamp_set()
796 priv->systime_flags |= tstamp_all | ptp_v2 | in stmmac_hwtstamp_set()
802 stmmac_config_hw_tstamping(priv, priv->ptpaddr, priv->systime_flags); in stmmac_hwtstamp_set()
804 memcpy(&priv->tstamp_config, &config, sizeof(config)); in stmmac_hwtstamp_set()
821 struct stmmac_priv *priv = netdev_priv(dev); in stmmac_hwtstamp_get() local
822 struct hwtstamp_config *config = &priv->tstamp_config; in stmmac_hwtstamp_get()
824 if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp)) in stmmac_hwtstamp_get()
841 int stmmac_init_tstamp_counter(struct stmmac_priv *priv, u32 systime_flags) in stmmac_init_tstamp_counter() argument
843 bool xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac; in stmmac_init_tstamp_counter()
848 if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp)) in stmmac_init_tstamp_counter()
851 stmmac_config_hw_tstamping(priv, priv->ptpaddr, systime_flags); in stmmac_init_tstamp_counter()
852 priv->systime_flags = systime_flags; in stmmac_init_tstamp_counter()
855 stmmac_config_sub_second_increment(priv, priv->ptpaddr, in stmmac_init_tstamp_counter()
856 priv->plat->clk_ptp_rate, in stmmac_init_tstamp_counter()
861 priv->sub_second_inc = sec_inc; in stmmac_init_tstamp_counter()
869 priv->default_addend = div_u64(temp, priv->plat->clk_ptp_rate); in stmmac_init_tstamp_counter()
870 stmmac_config_addend(priv, priv->ptpaddr, priv->default_addend); in stmmac_init_tstamp_counter()
876 stmmac_init_systime(priv, priv->ptpaddr, (u32)now.tv_sec, now.tv_nsec); in stmmac_init_tstamp_counter()
889 static int stmmac_init_ptp(struct stmmac_priv *priv) in stmmac_init_ptp() argument
891 bool xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac; in stmmac_init_ptp()
894 if (priv->plat->ptp_clk_freq_config) in stmmac_init_ptp()
895 priv->plat->ptp_clk_freq_config(priv); in stmmac_init_ptp()
897 ret = stmmac_init_tstamp_counter(priv, STMMAC_HWTS_ACTIVE); in stmmac_init_ptp()
901 priv->adv_ts = 0; in stmmac_init_ptp()
903 if (xmac && priv->dma_cap.atime_stamp) in stmmac_init_ptp()
904 priv->adv_ts = 1; in stmmac_init_ptp()
906 else if (priv->extend_desc && priv->dma_cap.atime_stamp) in stmmac_init_ptp()
907 priv->adv_ts = 1; in stmmac_init_ptp()
909 if (priv->dma_cap.time_stamp) in stmmac_init_ptp()
910 netdev_info(priv->dev, "IEEE 1588-2002 Timestamp supported\n"); in stmmac_init_ptp()
912 if (priv->adv_ts) in stmmac_init_ptp()
913 netdev_info(priv->dev, in stmmac_init_ptp()
916 priv->hwts_tx_en = 0; in stmmac_init_ptp()
917 priv->hwts_rx_en = 0; in stmmac_init_ptp()
922 static void stmmac_release_ptp(struct stmmac_priv *priv) in stmmac_release_ptp() argument
924 clk_disable_unprepare(priv->plat->clk_ptp_ref); in stmmac_release_ptp()
925 stmmac_ptp_unregister(priv); in stmmac_release_ptp()
934 static void stmmac_mac_flow_ctrl(struct stmmac_priv *priv, u32 duplex) in stmmac_mac_flow_ctrl() argument
936 u32 tx_cnt = priv->plat->tx_queues_to_use; in stmmac_mac_flow_ctrl()
938 stmmac_flow_ctrl(priv, priv->hw, duplex, priv->flow_ctrl, in stmmac_mac_flow_ctrl()
939 priv->pause, tx_cnt); in stmmac_mac_flow_ctrl()
946 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); in stmmac_validate() local
949 int tx_cnt = priv->plat->tx_queues_to_use; in stmmac_validate()
950 int max_speed = priv->plat->max_speed; in stmmac_validate()
969 } else if (priv->plat->has_gmac4) { in stmmac_validate()
974 } else if (priv->plat->has_xgmac) { in stmmac_validate()
1039 if (priv->hw->xpcs) in stmmac_validate()
1040 xpcs_validate(priv->hw->xpcs, supported, state); in stmmac_validate()
1049 static void stmmac_fpe_link_state_handle(struct stmmac_priv *priv, bool is_up) in stmmac_fpe_link_state_handle() argument
1051 struct stmmac_fpe_cfg *fpe_cfg = priv->plat->fpe_cfg; in stmmac_fpe_link_state_handle()
1057 stmmac_fpe_send_mpacket(priv, priv->ioaddr, fpe_cfg, in stmmac_fpe_link_state_handle()
1068 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); in stmmac_mac_link_down() local
1070 stmmac_mac_set(priv, priv->ioaddr, false); in stmmac_mac_link_down()
1071 priv->eee_active = false; in stmmac_mac_link_down()
1072 priv->tx_lpi_enabled = false; in stmmac_mac_link_down()
1073 priv->eee_enabled = stmmac_eee_init(priv); in stmmac_mac_link_down()
1074 stmmac_set_eee_pls(priv, priv->hw, false); in stmmac_mac_link_down()
1076 if (priv->dma_cap.fpesel) in stmmac_mac_link_down()
1077 stmmac_fpe_link_state_handle(priv, false); in stmmac_mac_link_down()
1086 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); in stmmac_mac_link_up() local
1089 old_ctrl = readl(priv->ioaddr + MAC_CTRL_REG); in stmmac_mac_link_up()
1090 ctrl = old_ctrl & ~priv->hw->link.speed_mask; in stmmac_mac_link_up()
1095 ctrl |= priv->hw->link.xgmii.speed10000; in stmmac_mac_link_up()
1098 ctrl |= priv->hw->link.xgmii.speed5000; in stmmac_mac_link_up()
1101 ctrl |= priv->hw->link.xgmii.speed2500; in stmmac_mac_link_up()
1109 ctrl |= priv->hw->link.xlgmii.speed100000; in stmmac_mac_link_up()
1112 ctrl |= priv->hw->link.xlgmii.speed50000; in stmmac_mac_link_up()
1115 ctrl |= priv->hw->link.xlgmii.speed40000; in stmmac_mac_link_up()
1118 ctrl |= priv->hw->link.xlgmii.speed25000; in stmmac_mac_link_up()
1121 ctrl |= priv->hw->link.xgmii.speed10000; in stmmac_mac_link_up()
1124 ctrl |= priv->hw->link.speed2500; in stmmac_mac_link_up()
1127 ctrl |= priv->hw->link.speed1000; in stmmac_mac_link_up()
1135 ctrl |= priv->hw->link.speed2500; in stmmac_mac_link_up()
1138 ctrl |= priv->hw->link.speed1000; in stmmac_mac_link_up()
1141 ctrl |= priv->hw->link.speed100; in stmmac_mac_link_up()
1144 ctrl |= priv->hw->link.speed10; in stmmac_mac_link_up()
1151 priv->speed = speed; in stmmac_mac_link_up()
1153 if (priv->plat->fix_mac_speed) in stmmac_mac_link_up()
1154 priv->plat->fix_mac_speed(priv->plat->bsp_priv, speed); in stmmac_mac_link_up()
1157 ctrl &= ~priv->hw->link.duplex; in stmmac_mac_link_up()
1159 ctrl |= priv->hw->link.duplex; in stmmac_mac_link_up()
1163 priv->flow_ctrl = FLOW_AUTO; in stmmac_mac_link_up()
1165 priv->flow_ctrl = FLOW_RX; in stmmac_mac_link_up()
1167 priv->flow_ctrl = FLOW_TX; in stmmac_mac_link_up()
1169 priv->flow_ctrl = FLOW_OFF; in stmmac_mac_link_up()
1171 stmmac_mac_flow_ctrl(priv, duplex); in stmmac_mac_link_up()
1174 writel(ctrl, priv->ioaddr + MAC_CTRL_REG); in stmmac_mac_link_up()
1176 stmmac_mac_set(priv, priv->ioaddr, true); in stmmac_mac_link_up()
1177 if (phy && priv->dma_cap.eee) { in stmmac_mac_link_up()
1178 priv->eee_active = in stmmac_mac_link_up()
1179 phy_init_eee(phy, !priv->plat->rx_clk_runs_in_lpi) >= 0; in stmmac_mac_link_up()
1180 priv->eee_enabled = stmmac_eee_init(priv); in stmmac_mac_link_up()
1181 priv->tx_lpi_enabled = priv->eee_enabled; in stmmac_mac_link_up()
1182 stmmac_set_eee_pls(priv, priv->hw, true); in stmmac_mac_link_up()
1185 if (priv->dma_cap.fpesel) in stmmac_mac_link_up()
1186 stmmac_fpe_link_state_handle(priv, true); in stmmac_mac_link_up()
1203 static void stmmac_check_pcs_mode(struct stmmac_priv *priv) in stmmac_check_pcs_mode() argument
1205 int interface = priv->plat->interface; in stmmac_check_pcs_mode()
1207 if (priv->dma_cap.pcs) { in stmmac_check_pcs_mode()
1212 netdev_dbg(priv->dev, "PCS RGMII support enabled\n"); in stmmac_check_pcs_mode()
1213 priv->hw->pcs = STMMAC_PCS_RGMII; in stmmac_check_pcs_mode()
1215 netdev_dbg(priv->dev, "PCS SGMII support enabled\n"); in stmmac_check_pcs_mode()
1216 priv->hw->pcs = STMMAC_PCS_SGMII; in stmmac_check_pcs_mode()
1231 struct stmmac_priv *priv = netdev_priv(dev); in stmmac_init_phy() local
1235 node = priv->plat->phylink_node; in stmmac_init_phy()
1238 ret = phylink_of_phy_connect(priv->phylink, node, 0); in stmmac_init_phy()
1244 int addr = priv->plat->phy_addr; in stmmac_init_phy()
1248 netdev_err(priv->dev, "no phy found\n"); in stmmac_init_phy()
1252 phydev = mdiobus_get_phy(priv->mii, addr); in stmmac_init_phy()
1254 netdev_err(priv->dev, "no phy at addr %d\n", addr); in stmmac_init_phy()
1258 ret = phylink_connect_phy(priv->phylink, phydev); in stmmac_init_phy()
1261 if (!priv->plat->pmt) { in stmmac_init_phy()
1264 phylink_ethtool_get_wol(priv->phylink, &wol); in stmmac_init_phy()
1265 device_set_wakeup_capable(priv->device, !!wol.supported); in stmmac_init_phy()
1266 device_set_wakeup_enable(priv->device, !!wol.wolopts); in stmmac_init_phy()
1272 static int stmmac_phy_setup(struct stmmac_priv *priv) in stmmac_phy_setup() argument
1274 struct stmmac_mdio_bus_data *mdio_bus_data = priv->plat->mdio_bus_data; in stmmac_phy_setup()
1275 struct fwnode_handle *fwnode = of_fwnode_handle(priv->plat->phylink_node); in stmmac_phy_setup()
1276 int mode = priv->plat->phy_interface; in stmmac_phy_setup()
1279 priv->phylink_config.dev = &priv->dev->dev; in stmmac_phy_setup()
1280 priv->phylink_config.type = PHYLINK_NETDEV; in stmmac_phy_setup()
1281 priv->phylink_config.pcs_poll = true; in stmmac_phy_setup()
1282 if (priv->plat->mdio_bus_data) in stmmac_phy_setup()
1283 priv->phylink_config.ovr_an_inband = in stmmac_phy_setup()
1287 fwnode = dev_fwnode(priv->device); in stmmac_phy_setup()
1289 phylink = phylink_create(&priv->phylink_config, fwnode, in stmmac_phy_setup()
1294 if (priv->hw->xpcs) in stmmac_phy_setup()
1295 phylink_set_pcs(phylink, &priv->hw->xpcs->pcs); in stmmac_phy_setup()
1297 priv->phylink = phylink; in stmmac_phy_setup()
1301 static void stmmac_display_rx_rings(struct stmmac_priv *priv) in stmmac_display_rx_rings() argument
1303 u32 rx_cnt = priv->plat->rx_queues_to_use; in stmmac_display_rx_rings()
1310 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; in stmmac_display_rx_rings()
1314 if (priv->extend_desc) { in stmmac_display_rx_rings()
1323 stmmac_display_ring(priv, head_rx, priv->dma_rx_size, true, in stmmac_display_rx_rings()
1328 static void stmmac_display_tx_rings(struct stmmac_priv *priv) in stmmac_display_tx_rings() argument
1330 u32 tx_cnt = priv->plat->tx_queues_to_use; in stmmac_display_tx_rings()
1337 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; in stmmac_display_tx_rings()
1341 if (priv->extend_desc) { in stmmac_display_tx_rings()
1352 stmmac_display_ring(priv, head_tx, priv->dma_tx_size, false, in stmmac_display_tx_rings()
1357 static void stmmac_display_rings(struct stmmac_priv *priv) in stmmac_display_rings() argument
1360 stmmac_display_rx_rings(priv); in stmmac_display_rings()
1363 stmmac_display_tx_rings(priv); in stmmac_display_rings()
1391 static void stmmac_clear_rx_descriptors(struct stmmac_priv *priv, u32 queue) in stmmac_clear_rx_descriptors() argument
1393 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; in stmmac_clear_rx_descriptors()
1397 for (i = 0; i < priv->dma_rx_size; i++) in stmmac_clear_rx_descriptors()
1398 if (priv->extend_desc) in stmmac_clear_rx_descriptors()
1399 stmmac_init_rx_desc(priv, &rx_q->dma_erx[i].basic, in stmmac_clear_rx_descriptors()
1400 priv->use_riwt, priv->mode, in stmmac_clear_rx_descriptors()
1401 (i == priv->dma_rx_size - 1), in stmmac_clear_rx_descriptors()
1402 priv->dma_buf_sz); in stmmac_clear_rx_descriptors()
1404 stmmac_init_rx_desc(priv, &rx_q->dma_rx[i], in stmmac_clear_rx_descriptors()
1405 priv->use_riwt, priv->mode, in stmmac_clear_rx_descriptors()
1406 (i == priv->dma_rx_size - 1), in stmmac_clear_rx_descriptors()
1407 priv->dma_buf_sz); in stmmac_clear_rx_descriptors()
1417 static void stmmac_clear_tx_descriptors(struct stmmac_priv *priv, u32 queue) in stmmac_clear_tx_descriptors() argument
1419 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; in stmmac_clear_tx_descriptors()
1423 for (i = 0; i < priv->dma_tx_size; i++) { in stmmac_clear_tx_descriptors()
1424 int last = (i == (priv->dma_tx_size - 1)); in stmmac_clear_tx_descriptors()
1427 if (priv->extend_desc) in stmmac_clear_tx_descriptors()
1434 stmmac_init_tx_desc(priv, p, priv->mode, last); in stmmac_clear_tx_descriptors()
1444 static void stmmac_clear_descriptors(struct stmmac_priv *priv) in stmmac_clear_descriptors() argument
1446 u32 rx_queue_cnt = priv->plat->rx_queues_to_use; in stmmac_clear_descriptors()
1447 u32 tx_queue_cnt = priv->plat->tx_queues_to_use; in stmmac_clear_descriptors()
1452 stmmac_clear_rx_descriptors(priv, queue); in stmmac_clear_descriptors()
1456 stmmac_clear_tx_descriptors(priv, queue); in stmmac_clear_descriptors()
1469 static int stmmac_init_rx_buffers(struct stmmac_priv *priv, struct dma_desc *p, in stmmac_init_rx_buffers() argument
1472 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; in stmmac_init_rx_buffers()
1479 buf->page_offset = stmmac_rx_offset(priv); in stmmac_init_rx_buffers()
1482 if (priv->sph && !buf->sec_page) { in stmmac_init_rx_buffers()
1488 stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, true); in stmmac_init_rx_buffers()
1491 stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, false); in stmmac_init_rx_buffers()
1496 stmmac_set_desc_addr(priv, p, buf->addr); in stmmac_init_rx_buffers()
1497 if (priv->dma_buf_sz == BUF_SIZE_16KiB) in stmmac_init_rx_buffers()
1498 stmmac_init_desc3(priv, p); in stmmac_init_rx_buffers()
1509 static void stmmac_free_rx_buffer(struct stmmac_priv *priv, u32 queue, int i) in stmmac_free_rx_buffer() argument
1511 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; in stmmac_free_rx_buffer()
1529 static void stmmac_free_tx_buffer(struct stmmac_priv *priv, u32 queue, int i) in stmmac_free_tx_buffer() argument
1531 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; in stmmac_free_tx_buffer()
1536 dma_unmap_page(priv->device, in stmmac_free_tx_buffer()
1541 dma_unmap_single(priv->device, in stmmac_free_tx_buffer()
1572 static void dma_free_rx_skbufs(struct stmmac_priv *priv, u32 queue) in dma_free_rx_skbufs() argument
1576 for (i = 0; i < priv->dma_rx_size; i++) in dma_free_rx_skbufs()
1577 stmmac_free_rx_buffer(priv, queue, i); in dma_free_rx_skbufs()
1580 static int stmmac_alloc_rx_buffers(struct stmmac_priv *priv, u32 queue, in stmmac_alloc_rx_buffers() argument
1583 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; in stmmac_alloc_rx_buffers()
1586 for (i = 0; i < priv->dma_rx_size; i++) { in stmmac_alloc_rx_buffers()
1590 if (priv->extend_desc) in stmmac_alloc_rx_buffers()
1595 ret = stmmac_init_rx_buffers(priv, p, i, flags, in stmmac_alloc_rx_buffers()
1611 static void dma_free_rx_xskbufs(struct stmmac_priv *priv, u32 queue) in dma_free_rx_xskbufs() argument
1613 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; in dma_free_rx_xskbufs()
1616 for (i = 0; i < priv->dma_rx_size; i++) { in dma_free_rx_xskbufs()
1627 static int stmmac_alloc_rx_buffers_zc(struct stmmac_priv *priv, u32 queue) in stmmac_alloc_rx_buffers_zc() argument
1629 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; in stmmac_alloc_rx_buffers_zc()
1632 for (i = 0; i < priv->dma_rx_size; i++) { in stmmac_alloc_rx_buffers_zc()
1637 if (priv->extend_desc) in stmmac_alloc_rx_buffers_zc()
1649 stmmac_set_desc_addr(priv, p, dma_addr); in stmmac_alloc_rx_buffers_zc()
1656 static struct xsk_buff_pool *stmmac_get_xsk_pool(struct stmmac_priv *priv, u32 queue) in stmmac_get_xsk_pool() argument
1658 if (!stmmac_xdp_is_enabled(priv) || !test_bit(queue, priv->af_xdp_zc_qps)) in stmmac_get_xsk_pool()
1661 return xsk_get_pool_from_qid(priv->dev, queue); in stmmac_get_xsk_pool()
1673 static int __init_dma_rx_desc_rings(struct stmmac_priv *priv, u32 queue, gfp_t flags) in __init_dma_rx_desc_rings() argument
1675 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; in __init_dma_rx_desc_rings()
1678 netif_dbg(priv, probe, priv->dev, in __init_dma_rx_desc_rings()
1682 stmmac_clear_rx_descriptors(priv, queue); in __init_dma_rx_desc_rings()
1686 rx_q->xsk_pool = stmmac_get_xsk_pool(priv, queue); in __init_dma_rx_desc_rings()
1692 netdev_info(priv->dev, in __init_dma_rx_desc_rings()
1700 netdev_info(priv->dev, in __init_dma_rx_desc_rings()
1709 stmmac_alloc_rx_buffers_zc(priv, queue); in __init_dma_rx_desc_rings()
1711 ret = stmmac_alloc_rx_buffers(priv, queue, flags); in __init_dma_rx_desc_rings()
1720 if (priv->mode == STMMAC_CHAIN_MODE) { in __init_dma_rx_desc_rings()
1721 if (priv->extend_desc) in __init_dma_rx_desc_rings()
1722 stmmac_mode_init(priv, rx_q->dma_erx, in __init_dma_rx_desc_rings()
1724 priv->dma_rx_size, 1); in __init_dma_rx_desc_rings()
1726 stmmac_mode_init(priv, rx_q->dma_rx, in __init_dma_rx_desc_rings()
1728 priv->dma_rx_size, 0); in __init_dma_rx_desc_rings()
1736 struct stmmac_priv *priv = netdev_priv(dev); in init_dma_rx_desc_rings() local
1737 u32 rx_count = priv->plat->rx_queues_to_use; in init_dma_rx_desc_rings()
1742 netif_dbg(priv, probe, priv->dev, in init_dma_rx_desc_rings()
1746 ret = __init_dma_rx_desc_rings(priv, queue, flags); in init_dma_rx_desc_rings()
1755 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; in init_dma_rx_desc_rings()
1758 dma_free_rx_xskbufs(priv, queue); in init_dma_rx_desc_rings()
1760 dma_free_rx_skbufs(priv, queue); in init_dma_rx_desc_rings()
1782 static int __init_dma_tx_desc_rings(struct stmmac_priv *priv, u32 queue) in __init_dma_tx_desc_rings() argument
1784 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; in __init_dma_tx_desc_rings()
1787 netif_dbg(priv, probe, priv->dev, in __init_dma_tx_desc_rings()
1792 if (priv->mode == STMMAC_CHAIN_MODE) { in __init_dma_tx_desc_rings()
1793 if (priv->extend_desc) in __init_dma_tx_desc_rings()
1794 stmmac_mode_init(priv, tx_q->dma_etx, in __init_dma_tx_desc_rings()
1796 priv->dma_tx_size, 1); in __init_dma_tx_desc_rings()
1798 stmmac_mode_init(priv, tx_q->dma_tx, in __init_dma_tx_desc_rings()
1800 priv->dma_tx_size, 0); in __init_dma_tx_desc_rings()
1803 tx_q->xsk_pool = stmmac_get_xsk_pool(priv, queue); in __init_dma_tx_desc_rings()
1805 for (i = 0; i < priv->dma_tx_size; i++) { in __init_dma_tx_desc_rings()
1808 if (priv->extend_desc) in __init_dma_tx_desc_rings()
1815 stmmac_clear_desc(priv, p); in __init_dma_tx_desc_rings()
1828 netdev_tx_reset_queue(netdev_get_tx_queue(priv->dev, queue)); in __init_dma_tx_desc_rings()
1835 struct stmmac_priv *priv = netdev_priv(dev); in init_dma_tx_desc_rings() local
1839 tx_queue_cnt = priv->plat->tx_queues_to_use; in init_dma_tx_desc_rings()
1842 __init_dma_tx_desc_rings(priv, queue); in init_dma_tx_desc_rings()
1857 struct stmmac_priv *priv = netdev_priv(dev); in init_dma_desc_rings() local
1866 stmmac_clear_descriptors(priv); in init_dma_desc_rings()
1868 if (netif_msg_hw(priv)) in init_dma_desc_rings()
1869 stmmac_display_rings(priv); in init_dma_desc_rings()
1879 static void dma_free_tx_skbufs(struct stmmac_priv *priv, u32 queue) in dma_free_tx_skbufs() argument
1881 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; in dma_free_tx_skbufs()
1886 for (i = 0; i < priv->dma_tx_size; i++) in dma_free_tx_skbufs()
1887 stmmac_free_tx_buffer(priv, queue, i); in dma_free_tx_skbufs()
1900 static void stmmac_free_tx_skbufs(struct stmmac_priv *priv) in stmmac_free_tx_skbufs() argument
1902 u32 tx_queue_cnt = priv->plat->tx_queues_to_use; in stmmac_free_tx_skbufs()
1906 dma_free_tx_skbufs(priv, queue); in stmmac_free_tx_skbufs()
1914 static void __free_dma_rx_desc_resources(struct stmmac_priv *priv, u32 queue) in __free_dma_rx_desc_resources() argument
1916 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; in __free_dma_rx_desc_resources()
1920 dma_free_rx_xskbufs(priv, queue); in __free_dma_rx_desc_resources()
1922 dma_free_rx_skbufs(priv, queue); in __free_dma_rx_desc_resources()
1928 if (!priv->extend_desc) in __free_dma_rx_desc_resources()
1929 dma_free_coherent(priv->device, priv->dma_rx_size * in __free_dma_rx_desc_resources()
1933 dma_free_coherent(priv->device, priv->dma_rx_size * in __free_dma_rx_desc_resources()
1945 static void free_dma_rx_desc_resources(struct stmmac_priv *priv) in free_dma_rx_desc_resources() argument
1947 u32 rx_count = priv->plat->rx_queues_to_use; in free_dma_rx_desc_resources()
1952 __free_dma_rx_desc_resources(priv, queue); in free_dma_rx_desc_resources()
1960 static void __free_dma_tx_desc_resources(struct stmmac_priv *priv, u32 queue) in __free_dma_tx_desc_resources() argument
1962 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; in __free_dma_tx_desc_resources()
1967 dma_free_tx_skbufs(priv, queue); in __free_dma_tx_desc_resources()
1969 if (priv->extend_desc) { in __free_dma_tx_desc_resources()
1980 size *= priv->dma_tx_size; in __free_dma_tx_desc_resources()
1982 dma_free_coherent(priv->device, size, addr, tx_q->dma_tx_phy); in __free_dma_tx_desc_resources()
1988 static void free_dma_tx_desc_resources(struct stmmac_priv *priv) in free_dma_tx_desc_resources() argument
1990 u32 tx_count = priv->plat->tx_queues_to_use; in free_dma_tx_desc_resources()
1995 __free_dma_tx_desc_resources(priv, queue); in free_dma_tx_desc_resources()
2007 static int __alloc_dma_rx_desc_resources(struct stmmac_priv *priv, u32 queue) in __alloc_dma_rx_desc_resources() argument
2009 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; in __alloc_dma_rx_desc_resources()
2010 struct stmmac_channel *ch = &priv->channel[queue]; in __alloc_dma_rx_desc_resources()
2011 bool xdp_prog = stmmac_xdp_is_enabled(priv); in __alloc_dma_rx_desc_resources()
2018 rx_q->priv_data = priv; in __alloc_dma_rx_desc_resources()
2021 pp_params.pool_size = priv->dma_rx_size; in __alloc_dma_rx_desc_resources()
2022 num_pages = DIV_ROUND_UP(priv->dma_buf_sz, PAGE_SIZE); in __alloc_dma_rx_desc_resources()
2024 pp_params.nid = dev_to_node(priv->device); in __alloc_dma_rx_desc_resources()
2025 pp_params.dev = priv->device; in __alloc_dma_rx_desc_resources()
2027 pp_params.offset = stmmac_rx_offset(priv); in __alloc_dma_rx_desc_resources()
2037 rx_q->buf_pool = kcalloc(priv->dma_rx_size, in __alloc_dma_rx_desc_resources()
2043 if (priv->extend_desc) { in __alloc_dma_rx_desc_resources()
2044 rx_q->dma_erx = dma_alloc_coherent(priv->device, in __alloc_dma_rx_desc_resources()
2045 priv->dma_rx_size * in __alloc_dma_rx_desc_resources()
2053 rx_q->dma_rx = dma_alloc_coherent(priv->device, in __alloc_dma_rx_desc_resources()
2054 priv->dma_rx_size * in __alloc_dma_rx_desc_resources()
2062 if (stmmac_xdp_is_enabled(priv) && in __alloc_dma_rx_desc_resources()
2063 test_bit(queue, priv->af_xdp_zc_qps)) in __alloc_dma_rx_desc_resources()
2068 ret = xdp_rxq_info_reg(&rx_q->xdp_rxq, priv->dev, in __alloc_dma_rx_desc_resources()
2072 netdev_err(priv->dev, "Failed to register xdp rxq info\n"); in __alloc_dma_rx_desc_resources()
2079 static int alloc_dma_rx_desc_resources(struct stmmac_priv *priv) in alloc_dma_rx_desc_resources() argument
2081 u32 rx_count = priv->plat->rx_queues_to_use; in alloc_dma_rx_desc_resources()
2087 ret = __alloc_dma_rx_desc_resources(priv, queue); in alloc_dma_rx_desc_resources()
2095 free_dma_rx_desc_resources(priv); in alloc_dma_rx_desc_resources()
2109 static int __alloc_dma_tx_desc_resources(struct stmmac_priv *priv, u32 queue) in __alloc_dma_tx_desc_resources() argument
2111 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; in __alloc_dma_tx_desc_resources()
2116 tx_q->priv_data = priv; in __alloc_dma_tx_desc_resources()
2118 tx_q->tx_skbuff_dma = kcalloc(priv->dma_tx_size, in __alloc_dma_tx_desc_resources()
2124 tx_q->tx_skbuff = kcalloc(priv->dma_tx_size, in __alloc_dma_tx_desc_resources()
2130 if (priv->extend_desc) in __alloc_dma_tx_desc_resources()
2137 size *= priv->dma_tx_size; in __alloc_dma_tx_desc_resources()
2139 addr = dma_alloc_coherent(priv->device, size, in __alloc_dma_tx_desc_resources()
2144 if (priv->extend_desc) in __alloc_dma_tx_desc_resources()
2154 static int alloc_dma_tx_desc_resources(struct stmmac_priv *priv) in alloc_dma_tx_desc_resources() argument
2156 u32 tx_count = priv->plat->tx_queues_to_use; in alloc_dma_tx_desc_resources()
2162 ret = __alloc_dma_tx_desc_resources(priv, queue); in alloc_dma_tx_desc_resources()
2170 free_dma_tx_desc_resources(priv); in alloc_dma_tx_desc_resources()
2182 static int alloc_dma_desc_resources(struct stmmac_priv *priv) in alloc_dma_desc_resources() argument
2185 int ret = alloc_dma_rx_desc_resources(priv); in alloc_dma_desc_resources()
2190 ret = alloc_dma_tx_desc_resources(priv); in alloc_dma_desc_resources()
2199 static void free_dma_desc_resources(struct stmmac_priv *priv) in free_dma_desc_resources() argument
2202 free_dma_tx_desc_resources(priv); in free_dma_desc_resources()
2207 free_dma_rx_desc_resources(priv); in free_dma_desc_resources()
2215 static void stmmac_mac_enable_rx_queues(struct stmmac_priv *priv) in stmmac_mac_enable_rx_queues() argument
2217 u32 rx_queues_count = priv->plat->rx_queues_to_use; in stmmac_mac_enable_rx_queues()
2222 mode = priv->plat->rx_queues_cfg[queue].mode_to_use; in stmmac_mac_enable_rx_queues()
2223 stmmac_rx_queue_enable(priv, priv->hw, mode, queue); in stmmac_mac_enable_rx_queues()
2234 static void stmmac_start_rx_dma(struct stmmac_priv *priv, u32 chan) in stmmac_start_rx_dma() argument
2236 netdev_dbg(priv->dev, "DMA RX processes started in channel %d\n", chan); in stmmac_start_rx_dma()
2237 stmmac_start_rx(priv, priv->ioaddr, chan); in stmmac_start_rx_dma()
2247 static void stmmac_start_tx_dma(struct stmmac_priv *priv, u32 chan) in stmmac_start_tx_dma() argument
2249 netdev_dbg(priv->dev, "DMA TX processes started in channel %d\n", chan); in stmmac_start_tx_dma()
2250 stmmac_start_tx(priv, priv->ioaddr, chan); in stmmac_start_tx_dma()
2260 static void stmmac_stop_rx_dma(struct stmmac_priv *priv, u32 chan) in stmmac_stop_rx_dma() argument
2262 netdev_dbg(priv->dev, "DMA RX processes stopped in channel %d\n", chan); in stmmac_stop_rx_dma()
2263 stmmac_stop_rx(priv, priv->ioaddr, chan); in stmmac_stop_rx_dma()
2273 static void stmmac_stop_tx_dma(struct stmmac_priv *priv, u32 chan) in stmmac_stop_tx_dma() argument
2275 netdev_dbg(priv->dev, "DMA TX processes stopped in channel %d\n", chan); in stmmac_stop_tx_dma()
2276 stmmac_stop_tx(priv, priv->ioaddr, chan); in stmmac_stop_tx_dma()
2279 static void stmmac_enable_all_dma_irq(struct stmmac_priv *priv) in stmmac_enable_all_dma_irq() argument
2281 u32 rx_channels_count = priv->plat->rx_queues_to_use; in stmmac_enable_all_dma_irq()
2282 u32 tx_channels_count = priv->plat->tx_queues_to_use; in stmmac_enable_all_dma_irq()
2287 struct stmmac_channel *ch = &priv->channel[chan]; in stmmac_enable_all_dma_irq()
2291 stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 1, 1); in stmmac_enable_all_dma_irq()
2302 static void stmmac_start_all_dma(struct stmmac_priv *priv) in stmmac_start_all_dma() argument
2304 u32 rx_channels_count = priv->plat->rx_queues_to_use; in stmmac_start_all_dma()
2305 u32 tx_channels_count = priv->plat->tx_queues_to_use; in stmmac_start_all_dma()
2309 stmmac_start_rx_dma(priv, chan); in stmmac_start_all_dma()
2312 stmmac_start_tx_dma(priv, chan); in stmmac_start_all_dma()
2321 static void stmmac_stop_all_dma(struct stmmac_priv *priv) in stmmac_stop_all_dma() argument
2323 u32 rx_channels_count = priv->plat->rx_queues_to_use; in stmmac_stop_all_dma()
2324 u32 tx_channels_count = priv->plat->tx_queues_to_use; in stmmac_stop_all_dma()
2328 stmmac_stop_rx_dma(priv, chan); in stmmac_stop_all_dma()
2331 stmmac_stop_tx_dma(priv, chan); in stmmac_stop_all_dma()
2340 static void stmmac_dma_operation_mode(struct stmmac_priv *priv) in stmmac_dma_operation_mode() argument
2342 u32 rx_channels_count = priv->plat->rx_queues_to_use; in stmmac_dma_operation_mode()
2343 u32 tx_channels_count = priv->plat->tx_queues_to_use; in stmmac_dma_operation_mode()
2344 int rxfifosz = priv->plat->rx_fifo_size; in stmmac_dma_operation_mode()
2345 int txfifosz = priv->plat->tx_fifo_size; in stmmac_dma_operation_mode()
2352 rxfifosz = priv->dma_cap.rx_fifo_size; in stmmac_dma_operation_mode()
2354 txfifosz = priv->dma_cap.tx_fifo_size; in stmmac_dma_operation_mode()
2360 if (priv->plat->force_thresh_dma_mode) { in stmmac_dma_operation_mode()
2363 } else if (priv->plat->force_sf_dma_mode || priv->plat->tx_coe) { in stmmac_dma_operation_mode()
2373 priv->xstats.threshold = SF_DMA_MODE; in stmmac_dma_operation_mode()
2381 struct stmmac_rx_queue *rx_q = &priv->rx_queue[chan]; in stmmac_dma_operation_mode()
2384 qmode = priv->plat->rx_queues_cfg[chan].mode_to_use; in stmmac_dma_operation_mode()
2386 stmmac_dma_rx_mode(priv, priv->ioaddr, rxmode, chan, in stmmac_dma_operation_mode()
2391 stmmac_set_dma_bfsize(priv, priv->ioaddr, in stmmac_dma_operation_mode()
2395 stmmac_set_dma_bfsize(priv, priv->ioaddr, in stmmac_dma_operation_mode()
2396 priv->dma_buf_sz, in stmmac_dma_operation_mode()
2402 qmode = priv->plat->tx_queues_cfg[chan].mode_to_use; in stmmac_dma_operation_mode()
2404 stmmac_dma_tx_mode(priv, priv->ioaddr, txmode, chan, in stmmac_dma_operation_mode()
2409 static bool stmmac_xdp_xmit_zc(struct stmmac_priv *priv, u32 queue, u32 budget) in stmmac_xdp_xmit_zc() argument
2411 struct netdev_queue *nq = netdev_get_tx_queue(priv->dev, queue); in stmmac_xdp_xmit_zc()
2412 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; in stmmac_xdp_xmit_zc()
2422 budget = min(budget, stmmac_tx_avail(priv, queue)); in stmmac_xdp_xmit_zc()
2431 if (unlikely(stmmac_tx_avail(priv, queue) < STMMAC_TX_XSK_AVAIL) || in stmmac_xdp_xmit_zc()
2432 !netif_carrier_ok(priv->dev)) { in stmmac_xdp_xmit_zc()
2440 if (likely(priv->extend_desc)) in stmmac_xdp_xmit_zc()
2464 stmmac_set_desc_addr(priv, tx_desc, dma_addr); in stmmac_xdp_xmit_zc()
2468 if (!priv->tx_coal_frames[queue]) in stmmac_xdp_xmit_zc()
2470 else if (tx_q->tx_count_frames % priv->tx_coal_frames[queue] == 0) in stmmac_xdp_xmit_zc()
2477 stmmac_set_tx_ic(priv, tx_desc); in stmmac_xdp_xmit_zc()
2478 priv->xstats.tx_set_ic_bit++; in stmmac_xdp_xmit_zc()
2481 stmmac_prepare_tx_desc(priv, tx_desc, 1, xdp_desc.len, in stmmac_xdp_xmit_zc()
2482 true, priv->mode, true, true, in stmmac_xdp_xmit_zc()
2485 stmmac_enable_dma_transmission(priv, priv->ioaddr); in stmmac_xdp_xmit_zc()
2487 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, priv->dma_tx_size); in stmmac_xdp_xmit_zc()
2492 stmmac_flush_tx_descriptors(priv, queue); in stmmac_xdp_xmit_zc()
2511 static int stmmac_tx_clean(struct stmmac_priv *priv, int budget, u32 queue) in stmmac_tx_clean() argument
2513 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; in stmmac_tx_clean()
2517 __netif_tx_lock_bh(netdev_get_tx_queue(priv->dev, queue)); in stmmac_tx_clean()
2519 priv->xstats.tx_clean++; in stmmac_tx_clean()
2526 while ((entry != tx_q->cur_tx) && count < priv->dma_tx_size) { in stmmac_tx_clean()
2544 if (priv->extend_desc) in stmmac_tx_clean()
2551 status = stmmac_tx_status(priv, &priv->dev->stats, in stmmac_tx_clean()
2552 &priv->xstats, p, priv->ioaddr); in stmmac_tx_clean()
2568 priv->dev->stats.tx_errors++; in stmmac_tx_clean()
2570 priv->dev->stats.tx_packets++; in stmmac_tx_clean()
2571 priv->xstats.tx_pkt_n++; in stmmac_tx_clean()
2572 priv->xstats.txq_stats[queue].tx_pkt_n++; in stmmac_tx_clean()
2575 stmmac_get_tx_hwtstamp(priv, p, skb); in stmmac_tx_clean()
2581 dma_unmap_page(priv->device, in stmmac_tx_clean()
2586 dma_unmap_single(priv->device, in stmmac_tx_clean()
2595 stmmac_clean_desc3(priv, tx_q, p); in stmmac_tx_clean()
2624 stmmac_release_tx_desc(priv, p, priv->mode); in stmmac_tx_clean()
2626 entry = STMMAC_GET_ENTRY(entry, priv->dma_tx_size); in stmmac_tx_clean()
2630 netdev_tx_completed_queue(netdev_get_tx_queue(priv->dev, queue), in stmmac_tx_clean()
2633 if (unlikely(netif_tx_queue_stopped(netdev_get_tx_queue(priv->dev, in stmmac_tx_clean()
2635 stmmac_tx_avail(priv, queue) > STMMAC_TX_THRESH(priv)) { in stmmac_tx_clean()
2637 netif_dbg(priv, tx_done, priv->dev, in stmmac_tx_clean()
2639 netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, queue)); in stmmac_tx_clean()
2656 work_done = stmmac_xdp_xmit_zc(priv, queue, in stmmac_tx_clean()
2664 if (priv->eee_enabled && !priv->tx_path_in_lpi_mode && in stmmac_tx_clean()
2665 priv->eee_sw_timer_en) { in stmmac_tx_clean()
2666 if (stmmac_enable_eee_mode(priv)) in stmmac_tx_clean()
2667 mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(priv->tx_lpi_timer)); in stmmac_tx_clean()
2673 STMMAC_COAL_TIMER(priv->tx_coal_timer[queue]), in stmmac_tx_clean()
2676 __netif_tx_unlock_bh(netdev_get_tx_queue(priv->dev, queue)); in stmmac_tx_clean()
2689 static void stmmac_tx_err(struct stmmac_priv *priv, u32 chan) in stmmac_tx_err() argument
2691 struct stmmac_tx_queue *tx_q = &priv->tx_queue[chan]; in stmmac_tx_err()
2693 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, chan)); in stmmac_tx_err()
2695 stmmac_stop_tx_dma(priv, chan); in stmmac_tx_err()
2696 dma_free_tx_skbufs(priv, chan); in stmmac_tx_err()
2697 stmmac_clear_tx_descriptors(priv, chan); in stmmac_tx_err()
2701 netdev_tx_reset_queue(netdev_get_tx_queue(priv->dev, chan)); in stmmac_tx_err()
2702 stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, in stmmac_tx_err()
2704 stmmac_start_tx_dma(priv, chan); in stmmac_tx_err()
2706 priv->dev->stats.tx_errors++; in stmmac_tx_err()
2707 netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, chan)); in stmmac_tx_err()
2720 static void stmmac_set_dma_operation_mode(struct stmmac_priv *priv, u32 txmode, in stmmac_set_dma_operation_mode() argument
2723 u8 rxqmode = priv->plat->rx_queues_cfg[chan].mode_to_use; in stmmac_set_dma_operation_mode()
2724 u8 txqmode = priv->plat->tx_queues_cfg[chan].mode_to_use; in stmmac_set_dma_operation_mode()
2725 u32 rx_channels_count = priv->plat->rx_queues_to_use; in stmmac_set_dma_operation_mode()
2726 u32 tx_channels_count = priv->plat->tx_queues_to_use; in stmmac_set_dma_operation_mode()
2727 int rxfifosz = priv->plat->rx_fifo_size; in stmmac_set_dma_operation_mode()
2728 int txfifosz = priv->plat->tx_fifo_size; in stmmac_set_dma_operation_mode()
2731 rxfifosz = priv->dma_cap.rx_fifo_size; in stmmac_set_dma_operation_mode()
2733 txfifosz = priv->dma_cap.tx_fifo_size; in stmmac_set_dma_operation_mode()
2739 stmmac_dma_rx_mode(priv, priv->ioaddr, rxmode, chan, rxfifosz, rxqmode); in stmmac_set_dma_operation_mode()
2740 stmmac_dma_tx_mode(priv, priv->ioaddr, txmode, chan, txfifosz, txqmode); in stmmac_set_dma_operation_mode()
2743 static bool stmmac_safety_feat_interrupt(struct stmmac_priv *priv) in stmmac_safety_feat_interrupt() argument
2747 ret = stmmac_safety_feat_irq_status(priv, priv->dev, in stmmac_safety_feat_interrupt()
2748 priv->ioaddr, priv->dma_cap.asp, &priv->sstats); in stmmac_safety_feat_interrupt()
2750 stmmac_global_err(priv); in stmmac_safety_feat_interrupt()
2757 static int stmmac_napi_check(struct stmmac_priv *priv, u32 chan, u32 dir) in stmmac_napi_check() argument
2759 int status = stmmac_dma_interrupt_status(priv, priv->ioaddr, in stmmac_napi_check()
2760 &priv->xstats, chan, dir); in stmmac_napi_check()
2761 struct stmmac_rx_queue *rx_q = &priv->rx_queue[chan]; in stmmac_napi_check()
2762 struct stmmac_tx_queue *tx_q = &priv->tx_queue[chan]; in stmmac_napi_check()
2763 struct stmmac_channel *ch = &priv->channel[chan]; in stmmac_napi_check()
2771 if ((status & handle_rx) && (chan < priv->plat->rx_queues_to_use)) { in stmmac_napi_check()
2774 stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 1, 0); in stmmac_napi_check()
2780 if ((status & handle_tx) && (chan < priv->plat->tx_queues_to_use)) { in stmmac_napi_check()
2783 stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 0, 1); in stmmac_napi_check()
2799 static void stmmac_dma_interrupt(struct stmmac_priv *priv) in stmmac_dma_interrupt() argument
2801 u32 tx_channel_count = priv->plat->tx_queues_to_use; in stmmac_dma_interrupt()
2802 u32 rx_channel_count = priv->plat->rx_queues_to_use; in stmmac_dma_interrupt()
2813 status[chan] = stmmac_napi_check(priv, chan, in stmmac_dma_interrupt()
2819 if (unlikely(priv->xstats.threshold != SF_DMA_MODE) && in stmmac_dma_interrupt()
2822 if (priv->plat->force_thresh_dma_mode) in stmmac_dma_interrupt()
2823 stmmac_set_dma_operation_mode(priv, in stmmac_dma_interrupt()
2828 stmmac_set_dma_operation_mode(priv, in stmmac_dma_interrupt()
2832 priv->xstats.threshold = tc; in stmmac_dma_interrupt()
2835 stmmac_tx_err(priv, chan); in stmmac_dma_interrupt()
2845 static void stmmac_mmc_setup(struct stmmac_priv *priv) in stmmac_mmc_setup() argument
2850 stmmac_mmc_intr_all_mask(priv, priv->mmcaddr); in stmmac_mmc_setup()
2852 if (priv->dma_cap.rmon) { in stmmac_mmc_setup()
2853 stmmac_mmc_ctrl(priv, priv->mmcaddr, mode); in stmmac_mmc_setup()
2854 memset(&priv->mmc, 0, sizeof(struct stmmac_counters)); in stmmac_mmc_setup()
2856 netdev_info(priv->dev, "No MAC Management Counters available\n"); in stmmac_mmc_setup()
2868 static int stmmac_get_hw_features(struct stmmac_priv *priv) in stmmac_get_hw_features() argument
2870 return stmmac_get_hw_feature(priv, priv->ioaddr, &priv->dma_cap) == 0; in stmmac_get_hw_features()
2880 static void stmmac_check_ether_addr(struct stmmac_priv *priv) in stmmac_check_ether_addr() argument
2882 if (!is_valid_ether_addr(priv->dev->dev_addr)) { in stmmac_check_ether_addr()
2883 stmmac_get_umac_addr(priv, priv->hw, priv->dev->dev_addr, 0); in stmmac_check_ether_addr()
2884 if (!is_valid_ether_addr(priv->dev->dev_addr)) in stmmac_check_ether_addr()
2885 eth_hw_addr_random(priv->dev); in stmmac_check_ether_addr()
2886 dev_info(priv->device, "device MAC address %pM\n", in stmmac_check_ether_addr()
2887 priv->dev->dev_addr); in stmmac_check_ether_addr()
2899 static int stmmac_init_dma_engine(struct stmmac_priv *priv) in stmmac_init_dma_engine() argument
2901 u32 rx_channels_count = priv->plat->rx_queues_to_use; in stmmac_init_dma_engine()
2902 u32 tx_channels_count = priv->plat->tx_queues_to_use; in stmmac_init_dma_engine()
2910 if (!priv->plat->dma_cfg || !priv->plat->dma_cfg->pbl) { in stmmac_init_dma_engine()
2911 dev_err(priv->device, "Invalid DMA configuration\n"); in stmmac_init_dma_engine()
2915 if (priv->extend_desc && (priv->mode == STMMAC_RING_MODE)) in stmmac_init_dma_engine()
2918 ret = stmmac_reset(priv, priv->ioaddr); in stmmac_init_dma_engine()
2920 dev_err(priv->device, "Failed to reset the dma\n"); in stmmac_init_dma_engine()
2925 stmmac_dma_init(priv, priv->ioaddr, priv->plat->dma_cfg, atds); in stmmac_init_dma_engine()
2927 if (priv->plat->axi) in stmmac_init_dma_engine()
2928 stmmac_axi(priv, priv->ioaddr, priv->plat->axi); in stmmac_init_dma_engine()
2932 stmmac_init_chan(priv, priv->ioaddr, priv->plat->dma_cfg, chan); in stmmac_init_dma_engine()
2933 stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 1, 1); in stmmac_init_dma_engine()
2938 rx_q = &priv->rx_queue[chan]; in stmmac_init_dma_engine()
2940 stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, in stmmac_init_dma_engine()
2946 stmmac_set_rx_tail_ptr(priv, priv->ioaddr, in stmmac_init_dma_engine()
2952 tx_q = &priv->tx_queue[chan]; in stmmac_init_dma_engine()
2954 stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, in stmmac_init_dma_engine()
2958 stmmac_set_tx_tail_ptr(priv, priv->ioaddr, in stmmac_init_dma_engine()
2965 static void stmmac_tx_timer_arm(struct stmmac_priv *priv, u32 queue) in stmmac_tx_timer_arm() argument
2967 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; in stmmac_tx_timer_arm()
2970 STMMAC_COAL_TIMER(priv->tx_coal_timer[queue]), in stmmac_tx_timer_arm()
2983 struct stmmac_priv *priv = tx_q->priv_data; in stmmac_tx_timer() local
2987 ch = &priv->channel[tx_q->queue_index]; in stmmac_tx_timer()
2994 stmmac_disable_dma_irq(priv, priv->ioaddr, ch->index, 0, 1); in stmmac_tx_timer()
3010 static void stmmac_init_coalesce(struct stmmac_priv *priv) in stmmac_init_coalesce() argument
3012 u32 tx_channel_count = priv->plat->tx_queues_to_use; in stmmac_init_coalesce()
3013 u32 rx_channel_count = priv->plat->rx_queues_to_use; in stmmac_init_coalesce()
3017 struct stmmac_tx_queue *tx_q = &priv->tx_queue[chan]; in stmmac_init_coalesce()
3019 priv->tx_coal_frames[chan] = STMMAC_TX_FRAMES; in stmmac_init_coalesce()
3020 priv->tx_coal_timer[chan] = STMMAC_COAL_TX_TIMER; in stmmac_init_coalesce()
3027 priv->rx_coal_frames[chan] = STMMAC_RX_FRAMES; in stmmac_init_coalesce()
3030 static void stmmac_set_rings_length(struct stmmac_priv *priv) in stmmac_set_rings_length() argument
3032 u32 rx_channels_count = priv->plat->rx_queues_to_use; in stmmac_set_rings_length()
3033 u32 tx_channels_count = priv->plat->tx_queues_to_use; in stmmac_set_rings_length()
3038 stmmac_set_tx_ring_len(priv, priv->ioaddr, in stmmac_set_rings_length()
3039 (priv->dma_tx_size - 1), chan); in stmmac_set_rings_length()
3043 stmmac_set_rx_ring_len(priv, priv->ioaddr, in stmmac_set_rings_length()
3044 (priv->dma_rx_size - 1), chan); in stmmac_set_rings_length()
3052 static void stmmac_set_tx_queue_weight(struct stmmac_priv *priv) in stmmac_set_tx_queue_weight() argument
3054 u32 tx_queues_count = priv->plat->tx_queues_to_use; in stmmac_set_tx_queue_weight()
3059 weight = priv->plat->tx_queues_cfg[queue].weight; in stmmac_set_tx_queue_weight()
3060 stmmac_set_mtl_tx_queue_weight(priv, priv->hw, weight, queue); in stmmac_set_tx_queue_weight()
3069 static void stmmac_configure_cbs(struct stmmac_priv *priv) in stmmac_configure_cbs() argument
3071 u32 tx_queues_count = priv->plat->tx_queues_to_use; in stmmac_configure_cbs()
3077 mode_to_use = priv->plat->tx_queues_cfg[queue].mode_to_use; in stmmac_configure_cbs()
3081 stmmac_config_cbs(priv, priv->hw, in stmmac_configure_cbs()
3082 priv->plat->tx_queues_cfg[queue].send_slope, in stmmac_configure_cbs()
3083 priv->plat->tx_queues_cfg[queue].idle_slope, in stmmac_configure_cbs()
3084 priv->plat->tx_queues_cfg[queue].high_credit, in stmmac_configure_cbs()
3085 priv->plat->tx_queues_cfg[queue].low_credit, in stmmac_configure_cbs()
3095 static void stmmac_rx_queue_dma_chan_map(struct stmmac_priv *priv) in stmmac_rx_queue_dma_chan_map() argument
3097 u32 rx_queues_count = priv->plat->rx_queues_to_use; in stmmac_rx_queue_dma_chan_map()
3102 chan = priv->plat->rx_queues_cfg[queue].chan; in stmmac_rx_queue_dma_chan_map()
3103 stmmac_map_mtl_to_dma(priv, priv->hw, queue, chan); in stmmac_rx_queue_dma_chan_map()
3112 static void stmmac_mac_config_rx_queues_prio(struct stmmac_priv *priv) in stmmac_mac_config_rx_queues_prio() argument
3114 u32 rx_queues_count = priv->plat->rx_queues_to_use; in stmmac_mac_config_rx_queues_prio()
3119 if (!priv->plat->rx_queues_cfg[queue].use_prio) in stmmac_mac_config_rx_queues_prio()
3122 prio = priv->plat->rx_queues_cfg[queue].prio; in stmmac_mac_config_rx_queues_prio()
3123 stmmac_rx_queue_prio(priv, priv->hw, prio, queue); in stmmac_mac_config_rx_queues_prio()
3132 static void stmmac_mac_config_tx_queues_prio(struct stmmac_priv *priv) in stmmac_mac_config_tx_queues_prio() argument
3134 u32 tx_queues_count = priv->plat->tx_queues_to_use; in stmmac_mac_config_tx_queues_prio()
3139 if (!priv->plat->tx_queues_cfg[queue].use_prio) in stmmac_mac_config_tx_queues_prio()
3142 prio = priv->plat->tx_queues_cfg[queue].prio; in stmmac_mac_config_tx_queues_prio()
3143 stmmac_tx_queue_prio(priv, priv->hw, prio, queue); in stmmac_mac_config_tx_queues_prio()
3152 static void stmmac_mac_config_rx_queues_routing(struct stmmac_priv *priv) in stmmac_mac_config_rx_queues_routing() argument
3154 u32 rx_queues_count = priv->plat->rx_queues_to_use; in stmmac_mac_config_rx_queues_routing()
3160 if (priv->plat->rx_queues_cfg[queue].pkt_route == 0x0) in stmmac_mac_config_rx_queues_routing()
3163 packet = priv->plat->rx_queues_cfg[queue].pkt_route; in stmmac_mac_config_rx_queues_routing()
3164 stmmac_rx_queue_routing(priv, priv->hw, packet, queue); in stmmac_mac_config_rx_queues_routing()
3168 static void stmmac_mac_config_rss(struct stmmac_priv *priv) in stmmac_mac_config_rss() argument
3170 if (!priv->dma_cap.rssen || !priv->plat->rss_en) { in stmmac_mac_config_rss()
3171 priv->rss.enable = false; in stmmac_mac_config_rss()
3175 if (priv->dev->features & NETIF_F_RXHASH) in stmmac_mac_config_rss()
3176 priv->rss.enable = true; in stmmac_mac_config_rss()
3178 priv->rss.enable = false; in stmmac_mac_config_rss()
3180 stmmac_rss_configure(priv, priv->hw, &priv->rss, in stmmac_mac_config_rss()
3181 priv->plat->rx_queues_to_use); in stmmac_mac_config_rss()
3189 static void stmmac_mtl_configuration(struct stmmac_priv *priv) in stmmac_mtl_configuration() argument
3191 u32 rx_queues_count = priv->plat->rx_queues_to_use; in stmmac_mtl_configuration()
3192 u32 tx_queues_count = priv->plat->tx_queues_to_use; in stmmac_mtl_configuration()
3195 stmmac_set_tx_queue_weight(priv); in stmmac_mtl_configuration()
3199 stmmac_prog_mtl_rx_algorithms(priv, priv->hw, in stmmac_mtl_configuration()
3200 priv->plat->rx_sched_algorithm); in stmmac_mtl_configuration()
3204 stmmac_prog_mtl_tx_algorithms(priv, priv->hw, in stmmac_mtl_configuration()
3205 priv->plat->tx_sched_algorithm); in stmmac_mtl_configuration()
3209 stmmac_configure_cbs(priv); in stmmac_mtl_configuration()
3212 stmmac_rx_queue_dma_chan_map(priv); in stmmac_mtl_configuration()
3215 stmmac_mac_enable_rx_queues(priv); in stmmac_mtl_configuration()
3219 stmmac_mac_config_rx_queues_prio(priv); in stmmac_mtl_configuration()
3223 stmmac_mac_config_tx_queues_prio(priv); in stmmac_mtl_configuration()
3227 stmmac_mac_config_rx_queues_routing(priv); in stmmac_mtl_configuration()
3231 stmmac_mac_config_rss(priv); in stmmac_mtl_configuration()
3234 static void stmmac_safety_feat_configuration(struct stmmac_priv *priv) in stmmac_safety_feat_configuration() argument
3236 if (priv->dma_cap.asp) { in stmmac_safety_feat_configuration()
3237 netdev_info(priv->dev, "Enabling Safety Features\n"); in stmmac_safety_feat_configuration()
3238 stmmac_safety_feat_config(priv, priv->ioaddr, priv->dma_cap.asp, in stmmac_safety_feat_configuration()
3239 priv->plat->safety_feat_cfg); in stmmac_safety_feat_configuration()
3241 netdev_info(priv->dev, "No Safety Features support found\n"); in stmmac_safety_feat_configuration()
3245 static int stmmac_fpe_start_wq(struct stmmac_priv *priv) in stmmac_fpe_start_wq() argument
3249 clear_bit(__FPE_TASK_SCHED, &priv->fpe_task_state); in stmmac_fpe_start_wq()
3250 clear_bit(__FPE_REMOVING, &priv->fpe_task_state); in stmmac_fpe_start_wq()
3252 name = priv->wq_name; in stmmac_fpe_start_wq()
3253 sprintf(name, "%s-fpe", priv->dev->name); in stmmac_fpe_start_wq()
3255 priv->fpe_wq = create_singlethread_workqueue(name); in stmmac_fpe_start_wq()
3256 if (!priv->fpe_wq) { in stmmac_fpe_start_wq()
3257 netdev_err(priv->dev, "%s: Failed to create workqueue\n", name); in stmmac_fpe_start_wq()
3261 netdev_info(priv->dev, "FPE workqueue start"); in stmmac_fpe_start_wq()
3281 struct stmmac_priv *priv = netdev_priv(dev); in stmmac_hw_setup() local
3282 u32 rx_cnt = priv->plat->rx_queues_to_use; in stmmac_hw_setup()
3283 u32 tx_cnt = priv->plat->tx_queues_to_use; in stmmac_hw_setup()
3289 ret = stmmac_init_dma_engine(priv); in stmmac_hw_setup()
3291 netdev_err(priv->dev, "%s: DMA engine initialization failed\n", in stmmac_hw_setup()
3297 stmmac_set_umac_addr(priv, priv->hw, dev->dev_addr, 0); in stmmac_hw_setup()
3300 if (priv->hw->pcs) { in stmmac_hw_setup()
3301 int speed = priv->plat->mac_port_sel_speed; in stmmac_hw_setup()
3305 priv->hw->ps = speed; in stmmac_hw_setup()
3307 dev_warn(priv->device, "invalid port speed\n"); in stmmac_hw_setup()
3308 priv->hw->ps = 0; in stmmac_hw_setup()
3313 stmmac_core_init(priv, priv->hw, dev); in stmmac_hw_setup()
3316 stmmac_mtl_configuration(priv); in stmmac_hw_setup()
3319 stmmac_safety_feat_configuration(priv); in stmmac_hw_setup()
3321 ret = stmmac_rx_ipc(priv, priv->hw); in stmmac_hw_setup()
3323 netdev_warn(priv->dev, "RX IPC Checksum Offload disabled\n"); in stmmac_hw_setup()
3324 priv->plat->rx_coe = STMMAC_RX_COE_NONE; in stmmac_hw_setup()
3325 priv->hw->rx_csum = 0; in stmmac_hw_setup()
3329 stmmac_mac_set(priv, priv->ioaddr, true); in stmmac_hw_setup()
3332 stmmac_dma_operation_mode(priv); in stmmac_hw_setup()
3334 stmmac_mmc_setup(priv); in stmmac_hw_setup()
3337 ret = clk_prepare_enable(priv->plat->clk_ptp_ref); in stmmac_hw_setup()
3339 netdev_warn(priv->dev, in stmmac_hw_setup()
3344 ret = stmmac_init_ptp(priv); in stmmac_hw_setup()
3346 netdev_warn(priv->dev, "PTP not supported by HW\n"); in stmmac_hw_setup()
3348 netdev_warn(priv->dev, "PTP init failed\n"); in stmmac_hw_setup()
3350 stmmac_ptp_register(priv); in stmmac_hw_setup()
3352 priv->eee_tw_timer = STMMAC_DEFAULT_TWT_LS; in stmmac_hw_setup()
3355 if (!priv->tx_lpi_timer) in stmmac_hw_setup()
3356 priv->tx_lpi_timer = eee_timer * 1000; in stmmac_hw_setup()
3358 if (priv->use_riwt) { in stmmac_hw_setup()
3362 if (!priv->rx_riwt[queue]) in stmmac_hw_setup()
3363 priv->rx_riwt[queue] = DEF_DMA_RIWT; in stmmac_hw_setup()
3365 stmmac_rx_watchdog(priv, priv->ioaddr, in stmmac_hw_setup()
3366 priv->rx_riwt[queue], queue); in stmmac_hw_setup()
3370 if (priv->hw->pcs) in stmmac_hw_setup()
3371 stmmac_pcs_ctrl_ane(priv, priv->ioaddr, 1, priv->hw->ps, 0); in stmmac_hw_setup()
3374 stmmac_set_rings_length(priv); in stmmac_hw_setup()
3377 if (priv->tso) { in stmmac_hw_setup()
3379 struct stmmac_tx_queue *tx_q = &priv->tx_queue[chan]; in stmmac_hw_setup()
3385 stmmac_enable_tso(priv, priv->ioaddr, 1, chan); in stmmac_hw_setup()
3390 sph_en = (priv->hw->rx_csum > 0) && priv->sph; in stmmac_hw_setup()
3392 stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan); in stmmac_hw_setup()
3396 if (priv->dma_cap.vlins) in stmmac_hw_setup()
3397 stmmac_enable_vlan(priv, priv->hw, STMMAC_VLAN_INSERT); in stmmac_hw_setup()
3401 struct stmmac_tx_queue *tx_q = &priv->tx_queue[chan]; in stmmac_hw_setup()
3404 stmmac_enable_tbs(priv, priv->ioaddr, enable, chan); in stmmac_hw_setup()
3408 netif_set_real_num_rx_queues(dev, priv->plat->rx_queues_to_use); in stmmac_hw_setup()
3409 netif_set_real_num_tx_queues(dev, priv->plat->tx_queues_to_use); in stmmac_hw_setup()
3412 stmmac_start_all_dma(priv); in stmmac_hw_setup()
3414 if (priv->dma_cap.fpesel) { in stmmac_hw_setup()
3415 stmmac_fpe_start_wq(priv); in stmmac_hw_setup()
3417 if (priv->plat->fpe_cfg->enable) in stmmac_hw_setup()
3418 stmmac_fpe_handshake(priv, true); in stmmac_hw_setup()
3426 struct stmmac_priv *priv = netdev_priv(dev); in stmmac_hw_teardown() local
3428 clk_disable_unprepare(priv->plat->clk_ptp_ref); in stmmac_hw_teardown()
3434 struct stmmac_priv *priv = netdev_priv(dev); in stmmac_free_irq() local
3439 irq_idx = priv->plat->tx_queues_to_use; in stmmac_free_irq()
3443 if (priv->tx_irq[j] > 0) { in stmmac_free_irq()
3444 irq_set_affinity_hint(priv->tx_irq[j], NULL); in stmmac_free_irq()
3445 free_irq(priv->tx_irq[j], &priv->tx_queue[j]); in stmmac_free_irq()
3448 irq_idx = priv->plat->rx_queues_to_use; in stmmac_free_irq()
3452 if (priv->rx_irq[j] > 0) { in stmmac_free_irq()
3453 irq_set_affinity_hint(priv->rx_irq[j], NULL); in stmmac_free_irq()
3454 free_irq(priv->rx_irq[j], &priv->rx_queue[j]); in stmmac_free_irq()
3458 if (priv->sfty_ue_irq > 0 && priv->sfty_ue_irq != dev->irq) in stmmac_free_irq()
3459 free_irq(priv->sfty_ue_irq, dev); in stmmac_free_irq()
3462 if (priv->sfty_ce_irq > 0 && priv->sfty_ce_irq != dev->irq) in stmmac_free_irq()
3463 free_irq(priv->sfty_ce_irq, dev); in stmmac_free_irq()
3466 if (priv->lpi_irq > 0 && priv->lpi_irq != dev->irq) in stmmac_free_irq()
3467 free_irq(priv->lpi_irq, dev); in stmmac_free_irq()
3470 if (priv->wol_irq > 0 && priv->wol_irq != dev->irq) in stmmac_free_irq()
3471 free_irq(priv->wol_irq, dev); in stmmac_free_irq()
3485 struct stmmac_priv *priv = netdev_priv(dev); in stmmac_request_irq_multi_msi() local
3494 int_name = priv->int_name_mac; in stmmac_request_irq_multi_msi()
3499 netdev_err(priv->dev, in stmmac_request_irq_multi_msi()
3509 priv->wol_irq_disabled = true; in stmmac_request_irq_multi_msi()
3510 if (priv->wol_irq > 0 && priv->wol_irq != dev->irq) { in stmmac_request_irq_multi_msi()
3511 int_name = priv->int_name_wol; in stmmac_request_irq_multi_msi()
3513 ret = request_irq(priv->wol_irq, in stmmac_request_irq_multi_msi()
3517 netdev_err(priv->dev, in stmmac_request_irq_multi_msi()
3519 __func__, priv->wol_irq, ret); in stmmac_request_irq_multi_msi()
3528 if (priv->lpi_irq > 0 && priv->lpi_irq != dev->irq) { in stmmac_request_irq_multi_msi()
3529 int_name = priv->int_name_lpi; in stmmac_request_irq_multi_msi()
3531 ret = request_irq(priv->lpi_irq, in stmmac_request_irq_multi_msi()
3535 netdev_err(priv->dev, in stmmac_request_irq_multi_msi()
3537 __func__, priv->lpi_irq, ret); in stmmac_request_irq_multi_msi()
3546 if (priv->sfty_ce_irq > 0 && priv->sfty_ce_irq != dev->irq) { in stmmac_request_irq_multi_msi()
3547 int_name = priv->int_name_sfty_ce; in stmmac_request_irq_multi_msi()
3549 ret = request_irq(priv->sfty_ce_irq, in stmmac_request_irq_multi_msi()
3553 netdev_err(priv->dev, in stmmac_request_irq_multi_msi()
3555 __func__, priv->sfty_ce_irq, ret); in stmmac_request_irq_multi_msi()
3564 if (priv->sfty_ue_irq > 0 && priv->sfty_ue_irq != dev->irq) { in stmmac_request_irq_multi_msi()
3565 int_name = priv->int_name_sfty_ue; in stmmac_request_irq_multi_msi()
3567 ret = request_irq(priv->sfty_ue_irq, in stmmac_request_irq_multi_msi()
3571 netdev_err(priv->dev, in stmmac_request_irq_multi_msi()
3573 __func__, priv->sfty_ue_irq, ret); in stmmac_request_irq_multi_msi()
3580 for (i = 0; i < priv->plat->rx_queues_to_use; i++) { in stmmac_request_irq_multi_msi()
3581 if (priv->rx_irq[i] == 0) in stmmac_request_irq_multi_msi()
3584 int_name = priv->int_name_rx_irq[i]; in stmmac_request_irq_multi_msi()
3586 ret = request_irq(priv->rx_irq[i], in stmmac_request_irq_multi_msi()
3588 0, int_name, &priv->rx_queue[i]); in stmmac_request_irq_multi_msi()
3590 netdev_err(priv->dev, in stmmac_request_irq_multi_msi()
3592 __func__, i, priv->rx_irq[i], ret); in stmmac_request_irq_multi_msi()
3599 irq_set_affinity_hint(priv->rx_irq[i], &cpu_mask); in stmmac_request_irq_multi_msi()
3603 for (i = 0; i < priv->plat->tx_queues_to_use; i++) { in stmmac_request_irq_multi_msi()
3604 if (priv->tx_irq[i] == 0) in stmmac_request_irq_multi_msi()
3607 int_name = priv->int_name_tx_irq[i]; in stmmac_request_irq_multi_msi()
3609 ret = request_irq(priv->tx_irq[i], in stmmac_request_irq_multi_msi()
3611 0, int_name, &priv->tx_queue[i]); in stmmac_request_irq_multi_msi()
3613 netdev_err(priv->dev, in stmmac_request_irq_multi_msi()
3615 __func__, i, priv->tx_irq[i], ret); in stmmac_request_irq_multi_msi()
3622 irq_set_affinity_hint(priv->tx_irq[i], &cpu_mask); in stmmac_request_irq_multi_msi()
3634 struct stmmac_priv *priv = netdev_priv(dev); in stmmac_request_irq_single() local
3641 netdev_err(priv->dev, in stmmac_request_irq_single()
3651 if (priv->wol_irq > 0 && priv->wol_irq != dev->irq) { in stmmac_request_irq_single()
3652 ret = request_irq(priv->wol_irq, stmmac_interrupt, in stmmac_request_irq_single()
3655 netdev_err(priv->dev, in stmmac_request_irq_single()
3657 __func__, priv->wol_irq, ret); in stmmac_request_irq_single()
3664 if (priv->lpi_irq > 0 && priv->lpi_irq != dev->irq) { in stmmac_request_irq_single()
3665 ret = request_irq(priv->lpi_irq, stmmac_interrupt, in stmmac_request_irq_single()
3668 netdev_err(priv->dev, in stmmac_request_irq_single()
3670 __func__, priv->lpi_irq, ret); in stmmac_request_irq_single()
3685 struct stmmac_priv *priv = netdev_priv(dev); in stmmac_request_irq() local
3689 if (priv->plat->multi_msi_en) in stmmac_request_irq()
3708 struct stmmac_priv *priv = netdev_priv(dev); in stmmac_open() local
3709 int mode = priv->plat->phy_interface; in stmmac_open()
3714 ret = pm_runtime_get_sync(priv->device); in stmmac_open()
3716 pm_runtime_put_noidle(priv->device); in stmmac_open()
3720 if (priv->hw->pcs != STMMAC_PCS_TBI && in stmmac_open()
3721 priv->hw->pcs != STMMAC_PCS_RTBI && in stmmac_open()
3722 (!priv->hw->xpcs || in stmmac_open()
3723 xpcs_get_an_mode(priv->hw->xpcs, mode) != DW_AN_C73)) { in stmmac_open()
3726 netdev_err(priv->dev, in stmmac_open()
3734 memset(&priv->xstats, 0, sizeof(struct stmmac_extra_stats)); in stmmac_open()
3735 priv->xstats.threshold = tc; in stmmac_open()
3737 bfsize = stmmac_set_16kib_bfsize(priv, dev->mtu); in stmmac_open()
3742 bfsize = stmmac_set_bfsize(dev->mtu, priv->dma_buf_sz); in stmmac_open()
3744 priv->dma_buf_sz = bfsize; in stmmac_open()
3747 priv->rx_copybreak = STMMAC_RX_COPYBREAK; in stmmac_open()
3749 if (!priv->dma_tx_size) in stmmac_open()
3750 priv->dma_tx_size = DMA_DEFAULT_TX_SIZE; in stmmac_open()
3751 if (!priv->dma_rx_size) in stmmac_open()
3752 priv->dma_rx_size = DMA_DEFAULT_RX_SIZE; in stmmac_open()
3755 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) { in stmmac_open()
3756 struct stmmac_tx_queue *tx_q = &priv->tx_queue[chan]; in stmmac_open()
3757 int tbs_en = priv->plat->tx_queues_cfg[chan].tbs_en; in stmmac_open()
3763 ret = alloc_dma_desc_resources(priv); in stmmac_open()
3765 netdev_err(priv->dev, "%s: DMA descriptors allocation failed\n", in stmmac_open()
3772 netdev_err(priv->dev, "%s: DMA descriptors initialization failed\n", in stmmac_open()
3777 if (priv->plat->serdes_powerup) { in stmmac_open()
3778 ret = priv->plat->serdes_powerup(dev, priv->plat->bsp_priv); in stmmac_open()
3780 netdev_err(priv->dev, "%s: Serdes powerup failed\n", in stmmac_open()
3788 netdev_err(priv->dev, "%s: Hw setup failed\n", __func__); in stmmac_open()
3792 stmmac_init_coalesce(priv); in stmmac_open()
3794 phylink_start(priv->phylink); in stmmac_open()
3796 phylink_speed_up(priv->phylink); in stmmac_open()
3802 stmmac_enable_all_queues(priv); in stmmac_open()
3803 netif_tx_start_all_queues(priv->dev); in stmmac_open()
3804 stmmac_enable_all_dma_irq(priv); in stmmac_open()
3809 phylink_stop(priv->phylink); in stmmac_open()
3811 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) in stmmac_open()
3812 hrtimer_cancel(&priv->tx_queue[chan].txtimer); in stmmac_open()
3816 free_dma_desc_resources(priv); in stmmac_open()
3818 phylink_disconnect_phy(priv->phylink); in stmmac_open()
3820 pm_runtime_put(priv->device); in stmmac_open()
3824 static void stmmac_fpe_stop_wq(struct stmmac_priv *priv) in stmmac_fpe_stop_wq() argument
3826 set_bit(__FPE_REMOVING, &priv->fpe_task_state); in stmmac_fpe_stop_wq()
3828 if (priv->fpe_wq) in stmmac_fpe_stop_wq()
3829 destroy_workqueue(priv->fpe_wq); in stmmac_fpe_stop_wq()
3831 netdev_info(priv->dev, "FPE workqueue stop"); in stmmac_fpe_stop_wq()
3842 struct stmmac_priv *priv = netdev_priv(dev); in stmmac_release() local
3847 if (device_may_wakeup(priv->device)) in stmmac_release()
3848 phylink_speed_down(priv->phylink, false); in stmmac_release()
3850 phylink_stop(priv->phylink); in stmmac_release()
3851 phylink_disconnect_phy(priv->phylink); in stmmac_release()
3853 stmmac_disable_all_queues(priv); in stmmac_release()
3855 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) in stmmac_release()
3856 hrtimer_cancel(&priv->tx_queue[chan].txtimer); in stmmac_release()
3861 if (priv->eee_enabled) { in stmmac_release()
3862 priv->tx_path_in_lpi_mode = false; in stmmac_release()
3863 del_timer_sync(&priv->eee_ctrl_timer); in stmmac_release()
3867 stmmac_stop_all_dma(priv); in stmmac_release()
3870 free_dma_desc_resources(priv); in stmmac_release()
3873 stmmac_mac_set(priv, priv->ioaddr, false); in stmmac_release()
3876 if (priv->plat->serdes_powerdown) in stmmac_release()
3877 priv->plat->serdes_powerdown(dev, priv->plat->bsp_priv); in stmmac_release()
3881 stmmac_release_ptp(priv); in stmmac_release()
3883 pm_runtime_put(priv->device); in stmmac_release()
3885 if (priv->dma_cap.fpesel) in stmmac_release()
3886 stmmac_fpe_stop_wq(priv); in stmmac_release()
3891 static bool stmmac_vlan_insert(struct stmmac_priv *priv, struct sk_buff *skb, in stmmac_vlan_insert() argument
3898 if (!priv->dma_cap.vlins) in stmmac_vlan_insert()
3914 if (stmmac_set_desc_vlan_tag(priv, p, tag, inner_tag, inner_type)) in stmmac_vlan_insert()
3917 stmmac_set_tx_owner(priv, p); in stmmac_vlan_insert()
3918 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, priv->dma_tx_size); in stmmac_vlan_insert()
3933 static void stmmac_tso_allocator(struct stmmac_priv *priv, dma_addr_t des, in stmmac_tso_allocator() argument
3936 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; in stmmac_tso_allocator()
3947 priv->dma_tx_size); in stmmac_tso_allocator()
3956 if (priv->dma_cap.addr64 <= 32) in stmmac_tso_allocator()
3959 stmmac_set_desc_addr(priv, desc, curr_addr); in stmmac_tso_allocator()
3964 stmmac_prepare_tso_tx_desc(priv, desc, 0, buff_size, in stmmac_tso_allocator()
3973 static void stmmac_flush_tx_descriptors(struct stmmac_priv *priv, int queue) in stmmac_flush_tx_descriptors() argument
3975 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; in stmmac_flush_tx_descriptors()
3978 if (likely(priv->extend_desc)) in stmmac_flush_tx_descriptors()
3992 stmmac_set_tx_tail_ptr(priv, priv->ioaddr, tx_q->tx_tail_addr, queue); in stmmac_flush_tx_descriptors()
4025 struct stmmac_priv *priv = netdev_priv(dev); in stmmac_tso_xmit() local
4037 tx_q = &priv->tx_queue[queue]; in stmmac_tso_xmit()
4050 if (unlikely(stmmac_tx_avail(priv, queue) < in stmmac_tso_xmit()
4053 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, in stmmac_tso_xmit()
4056 netdev_err(priv->dev, in stmmac_tso_xmit()
4074 stmmac_set_mss(priv, mss_desc, mss); in stmmac_tso_xmit()
4077 priv->dma_tx_size); in stmmac_tso_xmit()
4081 if (netif_msg_tx_queued(priv)) { in stmmac_tso_xmit()
4089 has_vlan = stmmac_vlan_insert(priv, skb, tx_q); in stmmac_tso_xmit()
4101 stmmac_set_desc_vlan(priv, first, STMMAC_VLAN_INSERT); in stmmac_tso_xmit()
4104 des = dma_map_single(priv->device, skb->data, skb_headlen(skb), in stmmac_tso_xmit()
4106 if (dma_mapping_error(priv->device, des)) in stmmac_tso_xmit()
4114 if (priv->dma_cap.addr64 <= 32) { in stmmac_tso_xmit()
4124 stmmac_set_desc_addr(priv, first, des); in stmmac_tso_xmit()
4130 stmmac_tso_allocator(priv, des, tmp_pay_len, (nfrags == 0), queue); in stmmac_tso_xmit()
4136 des = skb_frag_dma_map(priv->device, frag, 0, in stmmac_tso_xmit()
4139 if (dma_mapping_error(priv->device, des)) in stmmac_tso_xmit()
4142 stmmac_tso_allocator(priv, des, skb_frag_size(frag), in stmmac_tso_xmit()
4161 if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && priv->hwts_tx_en) in stmmac_tso_xmit()
4163 else if (!priv->tx_coal_frames[queue]) in stmmac_tso_xmit()
4165 else if (tx_packets > priv->tx_coal_frames[queue]) in stmmac_tso_xmit()
4168 priv->tx_coal_frames[queue]) < tx_packets) in stmmac_tso_xmit()
4180 stmmac_set_tx_ic(priv, desc); in stmmac_tso_xmit()
4181 priv->xstats.tx_set_ic_bit++; in stmmac_tso_xmit()
4189 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, priv->dma_tx_size); in stmmac_tso_xmit()
4191 if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) { in stmmac_tso_xmit()
4192 netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n", in stmmac_tso_xmit()
4194 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue)); in stmmac_tso_xmit()
4198 priv->xstats.tx_tso_frames++; in stmmac_tso_xmit()
4199 priv->xstats.tx_tso_nfrags += nfrags; in stmmac_tso_xmit()
4201 if (priv->sarc_type) in stmmac_tso_xmit()
4202 stmmac_set_desc_sarc(priv, first, priv->sarc_type); in stmmac_tso_xmit()
4207 priv->hwts_tx_en)) { in stmmac_tso_xmit()
4210 stmmac_enable_tx_timestamp(priv, first); in stmmac_tso_xmit()
4214 stmmac_prepare_tso_tx_desc(priv, first, 1, in stmmac_tso_xmit()
4228 stmmac_set_tx_owner(priv, mss_desc); in stmmac_tso_xmit()
4231 if (netif_msg_pktdata(priv)) { in stmmac_tso_xmit()
4241 stmmac_flush_tx_descriptors(priv, queue); in stmmac_tso_xmit()
4242 stmmac_tx_timer_arm(priv, queue); in stmmac_tso_xmit()
4247 dev_err(priv->device, "Tx dma map failed\n"); in stmmac_tso_xmit()
4249 priv->dev->stats.tx_dropped++; in stmmac_tso_xmit()
4264 struct stmmac_priv *priv = netdev_priv(dev); in stmmac_xmit() local
4277 tx_q = &priv->tx_queue[queue]; in stmmac_xmit()
4280 if (priv->tx_path_in_lpi_mode && priv->eee_sw_timer_en) in stmmac_xmit()
4281 stmmac_disable_eee_mode(priv); in stmmac_xmit()
4284 if (skb_is_gso(skb) && priv->tso) { in stmmac_xmit()
4287 if (priv->plat->has_gmac4 && (gso & SKB_GSO_UDP_L4)) in stmmac_xmit()
4291 if (unlikely(stmmac_tx_avail(priv, queue) < nfrags + 1)) { in stmmac_xmit()
4293 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, in stmmac_xmit()
4296 netdev_err(priv->dev, in stmmac_xmit()
4304 has_vlan = stmmac_vlan_insert(priv, skb, tx_q); in stmmac_xmit()
4312 if (likely(priv->extend_desc)) in stmmac_xmit()
4322 stmmac_set_desc_vlan(priv, first, STMMAC_VLAN_INSERT); in stmmac_xmit()
4324 enh_desc = priv->plat->enh_desc; in stmmac_xmit()
4327 is_jumbo = stmmac_is_jumbo_frm(priv, skb->len, enh_desc); in stmmac_xmit()
4330 entry = stmmac_jumbo_frm(priv, tx_q, skb, csum_insertion); in stmmac_xmit()
4340 entry = STMMAC_GET_ENTRY(entry, priv->dma_tx_size); in stmmac_xmit()
4343 if (likely(priv->extend_desc)) in stmmac_xmit()
4350 des = skb_frag_dma_map(priv->device, frag, 0, len, in stmmac_xmit()
4352 if (dma_mapping_error(priv->device, des)) in stmmac_xmit()
4357 stmmac_set_desc_addr(priv, desc, des); in stmmac_xmit()
4365 stmmac_prepare_tx_desc(priv, desc, 0, len, csum_insertion, in stmmac_xmit()
4366 priv->mode, 1, last_segment, skb->len); in stmmac_xmit()
4381 if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && priv->hwts_tx_en) in stmmac_xmit()
4383 else if (!priv->tx_coal_frames[queue]) in stmmac_xmit()
4385 else if (tx_packets > priv->tx_coal_frames[queue]) in stmmac_xmit()
4388 priv->tx_coal_frames[queue]) < tx_packets) in stmmac_xmit()
4394 if (likely(priv->extend_desc)) in stmmac_xmit()
4402 stmmac_set_tx_ic(priv, desc); in stmmac_xmit()
4403 priv->xstats.tx_set_ic_bit++; in stmmac_xmit()
4411 entry = STMMAC_GET_ENTRY(entry, priv->dma_tx_size); in stmmac_xmit()
4414 if (netif_msg_pktdata(priv)) { in stmmac_xmit()
4415 netdev_dbg(priv->dev, in stmmac_xmit()
4420 netdev_dbg(priv->dev, ">>> frame to be transmitted: "); in stmmac_xmit()
4424 if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) { in stmmac_xmit()
4425 netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n", in stmmac_xmit()
4427 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue)); in stmmac_xmit()
4432 if (priv->sarc_type) in stmmac_xmit()
4433 stmmac_set_desc_sarc(priv, first, priv->sarc_type); in stmmac_xmit()
4444 des = dma_map_single(priv->device, skb->data, in stmmac_xmit()
4446 if (dma_mapping_error(priv->device, des)) in stmmac_xmit()
4453 stmmac_set_desc_addr(priv, first, des); in stmmac_xmit()
4459 priv->hwts_tx_en)) { in stmmac_xmit()
4462 stmmac_enable_tx_timestamp(priv, first); in stmmac_xmit()
4466 stmmac_prepare_tx_desc(priv, first, 1, nopaged_len, in stmmac_xmit()
4467 csum_insertion, priv->mode, 0, last_segment, in stmmac_xmit()
4475 stmmac_set_desc_tbs(priv, tbs_desc, ts.tv_sec, ts.tv_nsec); in stmmac_xmit()
4478 stmmac_set_tx_owner(priv, first); in stmmac_xmit()
4482 stmmac_enable_dma_transmission(priv, priv->ioaddr); in stmmac_xmit()
4484 stmmac_flush_tx_descriptors(priv, queue); in stmmac_xmit()
4485 stmmac_tx_timer_arm(priv, queue); in stmmac_xmit()
4490 netdev_err(priv->dev, "Tx DMA map failed\n"); in stmmac_xmit()
4492 priv->dev->stats.tx_dropped++; in stmmac_xmit()
4521 static inline void stmmac_rx_refill(struct stmmac_priv *priv, u32 queue) in stmmac_rx_refill() argument
4523 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; in stmmac_rx_refill()
4524 int dirty = stmmac_rx_dirty(priv, queue); in stmmac_rx_refill()
4532 if (priv->extend_desc) in stmmac_rx_refill()
4543 if (priv->sph && !buf->sec_page) { in stmmac_rx_refill()
4553 stmmac_set_desc_addr(priv, p, buf->addr); in stmmac_rx_refill()
4554 if (priv->sph) in stmmac_rx_refill()
4555 stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, true); in stmmac_rx_refill()
4557 stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, false); in stmmac_rx_refill()
4558 stmmac_refill_desc3(priv, rx_q, p); in stmmac_rx_refill()
4561 rx_q->rx_count_frames += priv->rx_coal_frames[queue]; in stmmac_rx_refill()
4562 if (rx_q->rx_count_frames > priv->rx_coal_frames[queue]) in stmmac_rx_refill()
4565 use_rx_wd = !priv->rx_coal_frames[queue]; in stmmac_rx_refill()
4567 if (!priv->use_riwt) in stmmac_rx_refill()
4571 stmmac_set_rx_owner(priv, p, use_rx_wd); in stmmac_rx_refill()
4573 entry = STMMAC_GET_ENTRY(entry, priv->dma_rx_size); in stmmac_rx_refill()
4578 stmmac_set_rx_tail_ptr(priv, priv->ioaddr, rx_q->rx_tail_addr, queue); in stmmac_rx_refill()
4581 static unsigned int stmmac_rx_buf1_len(struct stmmac_priv *priv, in stmmac_rx_buf1_len() argument
4586 int coe = priv->hw->rx_csum; in stmmac_rx_buf1_len()
4589 if (priv->sph && len) in stmmac_rx_buf1_len()
4593 stmmac_get_rx_header_len(priv, p, &hlen); in stmmac_rx_buf1_len()
4594 if (priv->sph && hlen) { in stmmac_rx_buf1_len()
4595 priv->xstats.rx_split_hdr_pkt_n++; in stmmac_rx_buf1_len()
4601 return priv->dma_buf_sz; in stmmac_rx_buf1_len()
4603 plen = stmmac_get_rx_frame_len(priv, p, coe); in stmmac_rx_buf1_len()
4606 return min_t(unsigned int, priv->dma_buf_sz, plen); in stmmac_rx_buf1_len()
4609 static unsigned int stmmac_rx_buf2_len(struct stmmac_priv *priv, in stmmac_rx_buf2_len() argument
4613 int coe = priv->hw->rx_csum; in stmmac_rx_buf2_len()
4617 if (!priv->sph) in stmmac_rx_buf2_len()
4622 return priv->dma_buf_sz; in stmmac_rx_buf2_len()
4624 plen = stmmac_get_rx_frame_len(priv, p, coe); in stmmac_rx_buf2_len()
4630 static int stmmac_xdp_xmit_xdpf(struct stmmac_priv *priv, int queue, in stmmac_xdp_xmit_xdpf() argument
4633 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; in stmmac_xdp_xmit_xdpf()
4639 if (stmmac_tx_avail(priv, queue) < STMMAC_TX_THRESH(priv)) in stmmac_xdp_xmit_xdpf()
4642 if (likely(priv->extend_desc)) in stmmac_xdp_xmit_xdpf()
4650 dma_addr = dma_map_single(priv->device, xdpf->data, in stmmac_xdp_xmit_xdpf()
4652 if (dma_mapping_error(priv->device, dma_addr)) in stmmac_xdp_xmit_xdpf()
4661 dma_sync_single_for_device(priv->device, dma_addr, in stmmac_xdp_xmit_xdpf()
4675 stmmac_set_desc_addr(priv, tx_desc, dma_addr); in stmmac_xdp_xmit_xdpf()
4677 stmmac_prepare_tx_desc(priv, tx_desc, 1, xdpf->len, in stmmac_xdp_xmit_xdpf()
4678 true, priv->mode, true, true, in stmmac_xdp_xmit_xdpf()
4683 if (tx_q->tx_count_frames % priv->tx_coal_frames[queue] == 0) in stmmac_xdp_xmit_xdpf()
4690 stmmac_set_tx_ic(priv, tx_desc); in stmmac_xdp_xmit_xdpf()
4691 priv->xstats.tx_set_ic_bit++; in stmmac_xdp_xmit_xdpf()
4694 stmmac_enable_dma_transmission(priv, priv->ioaddr); in stmmac_xdp_xmit_xdpf()
4696 entry = STMMAC_GET_ENTRY(entry, priv->dma_tx_size); in stmmac_xdp_xmit_xdpf()
4702 static int stmmac_xdp_get_tx_queue(struct stmmac_priv *priv, in stmmac_xdp_get_tx_queue() argument
4710 while (index >= priv->plat->tx_queues_to_use) in stmmac_xdp_get_tx_queue()
4711 index -= priv->plat->tx_queues_to_use; in stmmac_xdp_get_tx_queue()
4716 static int stmmac_xdp_xmit_back(struct stmmac_priv *priv, in stmmac_xdp_xmit_back() argument
4728 queue = stmmac_xdp_get_tx_queue(priv, cpu); in stmmac_xdp_xmit_back()
4729 nq = netdev_get_tx_queue(priv->dev, queue); in stmmac_xdp_xmit_back()
4735 res = stmmac_xdp_xmit_xdpf(priv, queue, xdpf, false); in stmmac_xdp_xmit_back()
4737 stmmac_flush_tx_descriptors(priv, queue); in stmmac_xdp_xmit_back()
4744 static int __stmmac_xdp_run_prog(struct stmmac_priv *priv, in __stmmac_xdp_run_prog() argument
4757 res = stmmac_xdp_xmit_back(priv, xdp); in __stmmac_xdp_run_prog()
4760 if (xdp_do_redirect(priv->dev, xdp, prog) < 0) in __stmmac_xdp_run_prog()
4769 trace_xdp_exception(priv->dev, prog, act); in __stmmac_xdp_run_prog()
4779 static struct sk_buff *stmmac_xdp_run_prog(struct stmmac_priv *priv, in stmmac_xdp_run_prog() argument
4785 prog = READ_ONCE(priv->xdp_prog); in stmmac_xdp_run_prog()
4791 res = __stmmac_xdp_run_prog(priv, prog, xdp); in stmmac_xdp_run_prog()
4796 static void stmmac_finalize_xdp_rx(struct stmmac_priv *priv, in stmmac_finalize_xdp_rx() argument
4802 queue = stmmac_xdp_get_tx_queue(priv, cpu); in stmmac_finalize_xdp_rx()
4805 stmmac_tx_timer_arm(priv, queue); in stmmac_finalize_xdp_rx()
4832 static void stmmac_dispatch_skb_zc(struct stmmac_priv *priv, u32 queue, in stmmac_dispatch_skb_zc() argument
4836 struct stmmac_channel *ch = &priv->channel[queue]; in stmmac_dispatch_skb_zc()
4839 int coe = priv->hw->rx_csum; in stmmac_dispatch_skb_zc()
4845 priv->dev->stats.rx_dropped++; in stmmac_dispatch_skb_zc()
4849 stmmac_get_rx_hwtstamp(priv, p, np, skb); in stmmac_dispatch_skb_zc()
4850 stmmac_rx_vlan(priv->dev, skb); in stmmac_dispatch_skb_zc()
4851 skb->protocol = eth_type_trans(skb, priv->dev); in stmmac_dispatch_skb_zc()
4858 if (!stmmac_get_rx_hash(priv, p, &hash, &hash_type)) in stmmac_dispatch_skb_zc()
4864 priv->dev->stats.rx_packets++; in stmmac_dispatch_skb_zc()
4865 priv->dev->stats.rx_bytes += len; in stmmac_dispatch_skb_zc()
4868 static bool stmmac_rx_refill_zc(struct stmmac_priv *priv, u32 queue, u32 budget) in stmmac_rx_refill_zc() argument
4870 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; in stmmac_rx_refill_zc()
4875 budget = min(budget, stmmac_rx_dirty(priv, queue)); in stmmac_rx_refill_zc()
4890 if (priv->extend_desc) in stmmac_rx_refill_zc()
4896 stmmac_set_desc_addr(priv, rx_desc, dma_addr); in stmmac_rx_refill_zc()
4897 stmmac_set_desc_sec_addr(priv, rx_desc, 0, false); in stmmac_rx_refill_zc()
4898 stmmac_refill_desc3(priv, rx_q, rx_desc); in stmmac_rx_refill_zc()
4901 rx_q->rx_count_frames += priv->rx_coal_frames[queue]; in stmmac_rx_refill_zc()
4902 if (rx_q->rx_count_frames > priv->rx_coal_frames[queue]) in stmmac_rx_refill_zc()
4905 use_rx_wd = !priv->rx_coal_frames[queue]; in stmmac_rx_refill_zc()
4907 if (!priv->use_riwt) in stmmac_rx_refill_zc()
4911 stmmac_set_rx_owner(priv, rx_desc, use_rx_wd); in stmmac_rx_refill_zc()
4913 entry = STMMAC_GET_ENTRY(entry, priv->dma_rx_size); in stmmac_rx_refill_zc()
4920 stmmac_set_rx_tail_ptr(priv, priv->ioaddr, rx_q->rx_tail_addr, queue); in stmmac_rx_refill_zc()
4926 static int stmmac_rx_zc(struct stmmac_priv *priv, int limit, u32 queue) in stmmac_rx_zc() argument
4928 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; in stmmac_rx_zc()
4930 int dirty = stmmac_rx_dirty(priv, queue); in stmmac_rx_zc()
4938 if (netif_msg_rx_status(priv)) { in stmmac_rx_zc()
4941 netdev_dbg(priv->dev, "%s: descriptor ring:\n", __func__); in stmmac_rx_zc()
4942 if (priv->extend_desc) { in stmmac_rx_zc()
4950 stmmac_display_ring(priv, rx_head, priv->dma_rx_size, true, in stmmac_rx_zc()
4979 !stmmac_rx_refill_zc(priv, queue, dirty); in stmmac_rx_zc()
4983 if (priv->extend_desc) in stmmac_rx_zc()
4989 status = stmmac_rx_status(priv, &priv->dev->stats, in stmmac_rx_zc()
4990 &priv->xstats, p); in stmmac_rx_zc()
4997 priv->dma_rx_size); in stmmac_rx_zc()
5000 if (priv->extend_desc) in stmmac_rx_zc()
5011 if (priv->extend_desc) in stmmac_rx_zc()
5012 stmmac_rx_extended_status(priv, &priv->dev->stats, in stmmac_rx_zc()
5013 &priv->xstats, in stmmac_rx_zc()
5020 if (!priv->hwts_rx_en) in stmmac_rx_zc()
5021 priv->dev->stats.rx_errors++; in stmmac_rx_zc()
5041 buf1_len = stmmac_rx_buf1_len(priv, p, status, len); in stmmac_rx_zc()
5052 (likely(priv->synopsys_id >= DWMAC_CORE_4_00) || in stmmac_rx_zc()
5062 prog = READ_ONCE(priv->xdp_prog); in stmmac_rx_zc()
5063 res = __stmmac_xdp_run_prog(priv, prog, buf->xdp); in stmmac_rx_zc()
5067 stmmac_dispatch_skb_zc(priv, queue, p, np, buf->xdp); in stmmac_rx_zc()
5072 priv->dev->stats.rx_dropped++; in stmmac_rx_zc()
5091 stmmac_finalize_xdp_rx(priv, xdp_status); in stmmac_rx_zc()
5093 priv->xstats.rx_pkt_n += count; in stmmac_rx_zc()
5094 priv->xstats.rxq_stats[queue].rx_pkt_n += count; in stmmac_rx_zc()
5097 if (failure || stmmac_rx_dirty(priv, queue) > 0) in stmmac_rx_zc()
5116 static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue) in stmmac_rx() argument
5118 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; in stmmac_rx()
5119 struct stmmac_channel *ch = &priv->channel[queue]; in stmmac_rx()
5121 int status = 0, coe = priv->hw->rx_csum; in stmmac_rx()
5131 buf_sz = DIV_ROUND_UP(priv->dma_buf_sz, PAGE_SIZE) * PAGE_SIZE; in stmmac_rx()
5133 if (netif_msg_rx_status(priv)) { in stmmac_rx()
5136 netdev_dbg(priv->dev, "%s: descriptor ring:\n", __func__); in stmmac_rx()
5137 if (priv->extend_desc) { in stmmac_rx()
5145 stmmac_display_ring(priv, rx_head, priv->dma_rx_size, true, in stmmac_rx()
5176 if (priv->extend_desc) in stmmac_rx()
5182 status = stmmac_rx_status(priv, &priv->dev->stats, in stmmac_rx()
5183 &priv->xstats, p); in stmmac_rx()
5189 priv->dma_rx_size); in stmmac_rx()
5192 if (priv->extend_desc) in stmmac_rx()
5199 if (priv->extend_desc) in stmmac_rx()
5200 stmmac_rx_extended_status(priv, &priv->dev->stats, in stmmac_rx()
5201 &priv->xstats, rx_q->dma_erx + entry); in stmmac_rx()
5206 if (!priv->hwts_rx_en) in stmmac_rx()
5207 priv->dev->stats.rx_errors++; in stmmac_rx()
5225 buf1_len = stmmac_rx_buf1_len(priv, p, status, len); in stmmac_rx()
5227 buf2_len = stmmac_rx_buf2_len(priv, p, status, len); in stmmac_rx()
5238 (likely(priv->synopsys_id >= DWMAC_CORE_4_00) || in stmmac_rx()
5251 dma_sync_single_for_cpu(priv->device, buf->addr, in stmmac_rx()
5260 skb = stmmac_xdp_run_prog(priv, &xdp); in stmmac_rx()
5277 priv->dev->stats.rx_dropped++; in stmmac_rx()
5306 priv->dev->stats.rx_dropped++; in stmmac_rx()
5319 dma_sync_single_for_cpu(priv->device, buf->addr, in stmmac_rx()
5323 priv->dma_buf_sz); in stmmac_rx()
5331 dma_sync_single_for_cpu(priv->device, buf->sec_addr, in stmmac_rx()
5335 priv->dma_buf_sz); in stmmac_rx()
5350 stmmac_get_rx_hwtstamp(priv, p, np, skb); in stmmac_rx()
5351 stmmac_rx_vlan(priv->dev, skb); in stmmac_rx()
5352 skb->protocol = eth_type_trans(skb, priv->dev); in stmmac_rx()
5359 if (!stmmac_get_rx_hash(priv, p, &hash, &hash_type)) in stmmac_rx()
5366 priv->dev->stats.rx_packets++; in stmmac_rx()
5367 priv->dev->stats.rx_bytes += len; in stmmac_rx()
5378 stmmac_finalize_xdp_rx(priv, xdp_status); in stmmac_rx()
5380 stmmac_rx_refill(priv, queue); in stmmac_rx()
5382 priv->xstats.rx_pkt_n += count; in stmmac_rx()
5383 priv->xstats.rxq_stats[queue].rx_pkt_n += count; in stmmac_rx()
5392 struct stmmac_priv *priv = ch->priv_data; in stmmac_napi_poll_rx() local
5396 priv->xstats.napi_poll++; in stmmac_napi_poll_rx()
5398 work_done = stmmac_rx(priv, budget, chan); in stmmac_napi_poll_rx()
5403 stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 1, 0); in stmmac_napi_poll_rx()
5414 struct stmmac_priv *priv = ch->priv_data; in stmmac_napi_poll_tx() local
5418 priv->xstats.napi_poll++; in stmmac_napi_poll_tx()
5420 work_done = stmmac_tx_clean(priv, budget, chan); in stmmac_napi_poll_tx()
5427 stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 0, 1); in stmmac_napi_poll_tx()
5438 struct stmmac_priv *priv = ch->priv_data; in stmmac_napi_poll_rxtx() local
5442 priv->xstats.napi_poll++; in stmmac_napi_poll_rxtx()
5444 tx_done = stmmac_tx_clean(priv, budget, chan); in stmmac_napi_poll_rxtx()
5447 rx_done = stmmac_rx_zc(priv, budget, chan); in stmmac_napi_poll_rxtx()
5465 stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 1, 1); in stmmac_napi_poll_rxtx()
5483 struct stmmac_priv *priv = netdev_priv(dev); in stmmac_tx_timeout() local
5485 stmmac_global_err(priv); in stmmac_tx_timeout()
5499 struct stmmac_priv *priv = netdev_priv(dev); in stmmac_set_rx_mode() local
5501 stmmac_set_filter(priv, priv->hw, dev); in stmmac_set_rx_mode()
5517 struct stmmac_priv *priv = netdev_priv(dev); in stmmac_change_mtu() local
5518 int txfifosz = priv->plat->tx_fifo_size; in stmmac_change_mtu()
5522 txfifosz = priv->dma_cap.tx_fifo_size; in stmmac_change_mtu()
5524 txfifosz /= priv->plat->tx_queues_to_use; in stmmac_change_mtu()
5527 netdev_err(priv->dev, "must be stopped to change its MTU\n"); in stmmac_change_mtu()
5531 if (stmmac_xdp_is_enabled(priv) && new_mtu > ETH_DATA_LEN) { in stmmac_change_mtu()
5532 netdev_dbg(priv->dev, "Jumbo frames not supported for XDP\n"); in stmmac_change_mtu()
5552 struct stmmac_priv *priv = netdev_priv(dev); in stmmac_fix_features() local
5554 if (priv->plat->rx_coe == STMMAC_RX_COE_NONE) in stmmac_fix_features()
5557 if (!priv->plat->tx_coe) in stmmac_fix_features()
5565 if (priv->plat->bugged_jumbo && (dev->mtu > ETH_DATA_LEN)) in stmmac_fix_features()
5569 if ((priv->plat->tso_en) && (priv->dma_cap.tsoen)) { in stmmac_fix_features()
5571 priv->tso = true; in stmmac_fix_features()
5573 priv->tso = false; in stmmac_fix_features()
5582 struct stmmac_priv *priv = netdev_priv(netdev); in stmmac_set_features() local
5586 priv->hw->rx_csum = priv->plat->rx_coe; in stmmac_set_features()
5588 priv->hw->rx_csum = 0; in stmmac_set_features()
5592 stmmac_rx_ipc(priv, priv->hw); in stmmac_set_features()
5594 if (priv->sph_cap) { in stmmac_set_features()
5595 bool sph_en = (priv->hw->rx_csum > 0) && priv->sph; in stmmac_set_features()
5598 for (chan = 0; chan < priv->plat->rx_queues_to_use; chan++) in stmmac_set_features()
5599 stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan); in stmmac_set_features()
5605 static void stmmac_fpe_event_status(struct stmmac_priv *priv, int status) in stmmac_fpe_event_status() argument
5607 struct stmmac_fpe_cfg *fpe_cfg = priv->plat->fpe_cfg; in stmmac_fpe_event_status()
5622 stmmac_fpe_send_mpacket(priv, priv->ioaddr, in stmmac_fpe_event_status()
5641 if (!test_bit(__FPE_REMOVING, &priv->fpe_task_state) && in stmmac_fpe_event_status()
5642 !test_and_set_bit(__FPE_TASK_SCHED, &priv->fpe_task_state) && in stmmac_fpe_event_status()
5643 priv->fpe_wq) { in stmmac_fpe_event_status()
5644 queue_work(priv->fpe_wq, &priv->fpe_task); in stmmac_fpe_event_status()
5648 static void stmmac_common_interrupt(struct stmmac_priv *priv) in stmmac_common_interrupt() argument
5650 u32 rx_cnt = priv->plat->rx_queues_to_use; in stmmac_common_interrupt()
5651 u32 tx_cnt = priv->plat->tx_queues_to_use; in stmmac_common_interrupt()
5656 xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac; in stmmac_common_interrupt()
5659 if (priv->irq_wake) in stmmac_common_interrupt()
5660 pm_wakeup_event(priv->device, 0); in stmmac_common_interrupt()
5662 if (priv->dma_cap.estsel) in stmmac_common_interrupt()
5663 stmmac_est_irq_status(priv, priv->ioaddr, priv->dev, in stmmac_common_interrupt()
5664 &priv->xstats, tx_cnt); in stmmac_common_interrupt()
5666 if (priv->dma_cap.fpesel) { in stmmac_common_interrupt()
5667 int status = stmmac_fpe_irq_status(priv, priv->ioaddr, in stmmac_common_interrupt()
5668 priv->dev); in stmmac_common_interrupt()
5670 stmmac_fpe_event_status(priv, status); in stmmac_common_interrupt()
5674 if ((priv->plat->has_gmac) || xmac) { in stmmac_common_interrupt()
5675 int status = stmmac_host_irq_status(priv, priv->hw, &priv->xstats); in stmmac_common_interrupt()
5680 priv->tx_path_in_lpi_mode = true; in stmmac_common_interrupt()
5682 priv->tx_path_in_lpi_mode = false; in stmmac_common_interrupt()
5686 status = stmmac_host_mtl_irq_status(priv, priv->hw, in stmmac_common_interrupt()
5691 if (priv->hw->pcs) { in stmmac_common_interrupt()
5692 if (priv->xstats.pcs_link) in stmmac_common_interrupt()
5693 netif_carrier_on(priv->dev); in stmmac_common_interrupt()
5695 netif_carrier_off(priv->dev); in stmmac_common_interrupt()
5698 stmmac_timestamp_interrupt(priv, priv); in stmmac_common_interrupt()
5716 struct stmmac_priv *priv = netdev_priv(dev); in stmmac_interrupt() local
5719 if (test_bit(STMMAC_DOWN, &priv->state)) in stmmac_interrupt()
5723 if (stmmac_safety_feat_interrupt(priv)) in stmmac_interrupt()
5727 stmmac_common_interrupt(priv); in stmmac_interrupt()
5730 stmmac_dma_interrupt(priv); in stmmac_interrupt()
5738 struct stmmac_priv *priv = netdev_priv(dev); in stmmac_mac_interrupt() local
5741 netdev_err(priv->dev, "%s: invalid dev pointer\n", __func__); in stmmac_mac_interrupt()
5746 if (test_bit(STMMAC_DOWN, &priv->state)) in stmmac_mac_interrupt()
5750 stmmac_common_interrupt(priv); in stmmac_mac_interrupt()
5758 struct stmmac_priv *priv = netdev_priv(dev); in stmmac_safety_interrupt() local
5761 netdev_err(priv->dev, "%s: invalid dev pointer\n", __func__); in stmmac_safety_interrupt()
5766 if (test_bit(STMMAC_DOWN, &priv->state)) in stmmac_safety_interrupt()
5770 stmmac_safety_feat_interrupt(priv); in stmmac_safety_interrupt()
5779 struct stmmac_priv *priv; in stmmac_msi_intr_tx() local
5782 priv = container_of(tx_q, struct stmmac_priv, tx_queue[chan]); in stmmac_msi_intr_tx()
5785 netdev_err(priv->dev, "%s: invalid dev pointer\n", __func__); in stmmac_msi_intr_tx()
5790 if (test_bit(STMMAC_DOWN, &priv->state)) in stmmac_msi_intr_tx()
5793 status = stmmac_napi_check(priv, chan, DMA_DIR_TX); in stmmac_msi_intr_tx()
5797 if (unlikely(priv->xstats.threshold != SF_DMA_MODE) && in stmmac_msi_intr_tx()
5800 if (priv->plat->force_thresh_dma_mode) in stmmac_msi_intr_tx()
5801 stmmac_set_dma_operation_mode(priv, in stmmac_msi_intr_tx()
5806 stmmac_set_dma_operation_mode(priv, in stmmac_msi_intr_tx()
5810 priv->xstats.threshold = tc; in stmmac_msi_intr_tx()
5813 stmmac_tx_err(priv, chan); in stmmac_msi_intr_tx()
5823 struct stmmac_priv *priv; in stmmac_msi_intr_rx() local
5825 priv = container_of(rx_q, struct stmmac_priv, rx_queue[chan]); in stmmac_msi_intr_rx()
5828 netdev_err(priv->dev, "%s: invalid dev pointer\n", __func__); in stmmac_msi_intr_rx()
5833 if (test_bit(STMMAC_DOWN, &priv->state)) in stmmac_msi_intr_rx()
5836 stmmac_napi_check(priv, chan, DMA_DIR_RX); in stmmac_msi_intr_rx()
5847 struct stmmac_priv *priv = netdev_priv(dev); in stmmac_poll_controller() local
5851 if (test_bit(STMMAC_DOWN, &priv->state)) in stmmac_poll_controller()
5854 if (priv->plat->multi_msi_en) { in stmmac_poll_controller()
5855 for (i = 0; i < priv->plat->rx_queues_to_use; i++) in stmmac_poll_controller()
5856 stmmac_msi_intr_rx(0, &priv->rx_queue[i]); in stmmac_poll_controller()
5858 for (i = 0; i < priv->plat->tx_queues_to_use; i++) in stmmac_poll_controller()
5859 stmmac_msi_intr_tx(0, &priv->tx_queue[i]); in stmmac_poll_controller()
5879 struct stmmac_priv *priv = netdev_priv (dev); in stmmac_ioctl() local
5889 ret = phylink_mii_ioctl(priv->phylink, rq, cmd); in stmmac_ioctl()
5907 struct stmmac_priv *priv = cb_priv; in stmmac_setup_tc_block_cb() local
5910 if (!tc_cls_can_offload_and_chain0(priv->dev, type_data)) in stmmac_setup_tc_block_cb()
5913 __stmmac_disable_all_queues(priv); in stmmac_setup_tc_block_cb()
5917 ret = stmmac_tc_setup_cls_u32(priv, priv, type_data); in stmmac_setup_tc_block_cb()
5920 ret = stmmac_tc_setup_cls(priv, priv, type_data); in stmmac_setup_tc_block_cb()
5926 stmmac_enable_all_queues(priv); in stmmac_setup_tc_block_cb()
5935 struct stmmac_priv *priv = netdev_priv(ndev); in stmmac_setup_tc() local
5942 priv, priv, true); in stmmac_setup_tc()
5944 return stmmac_tc_setup_cbs(priv, priv, type_data); in stmmac_setup_tc()
5946 return stmmac_tc_setup_taprio(priv, priv, type_data); in stmmac_setup_tc()
5948 return stmmac_tc_setup_etf(priv, priv, type_data); in stmmac_setup_tc()
5974 struct stmmac_priv *priv = netdev_priv(ndev); in stmmac_set_mac_address() local
5977 ret = pm_runtime_get_sync(priv->device); in stmmac_set_mac_address()
5979 pm_runtime_put_noidle(priv->device); in stmmac_set_mac_address()
5987 stmmac_set_umac_addr(priv, priv->hw, ndev->dev_addr, 0); in stmmac_set_mac_address()
5990 pm_runtime_put(priv->device); in stmmac_set_mac_address()
6031 struct stmmac_priv *priv = netdev_priv(dev); in stmmac_rings_status_show() local
6032 u32 rx_count = priv->plat->rx_queues_to_use; in stmmac_rings_status_show()
6033 u32 tx_count = priv->plat->tx_queues_to_use; in stmmac_rings_status_show()
6040 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; in stmmac_rings_status_show()
6044 if (priv->extend_desc) { in stmmac_rings_status_show()
6047 priv->dma_rx_size, 1, seq, rx_q->dma_rx_phy); in stmmac_rings_status_show()
6051 priv->dma_rx_size, 0, seq, rx_q->dma_rx_phy); in stmmac_rings_status_show()
6056 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; in stmmac_rings_status_show()
6060 if (priv->extend_desc) { in stmmac_rings_status_show()
6063 priv->dma_tx_size, 1, seq, tx_q->dma_tx_phy); in stmmac_rings_status_show()
6067 priv->dma_tx_size, 0, seq, tx_q->dma_tx_phy); in stmmac_rings_status_show()
6078 struct stmmac_priv *priv = netdev_priv(dev); in stmmac_dma_cap_show() local
6080 if (!priv->hw_cap_support) { in stmmac_dma_cap_show()
6090 (priv->dma_cap.mbps_10_100) ? "Y" : "N"); in stmmac_dma_cap_show()
6092 (priv->dma_cap.mbps_1000) ? "Y" : "N"); in stmmac_dma_cap_show()
6094 (priv->dma_cap.half_duplex) ? "Y" : "N"); in stmmac_dma_cap_show()
6096 (priv->dma_cap.hash_filter) ? "Y" : "N"); in stmmac_dma_cap_show()
6098 (priv->dma_cap.multi_addr) ? "Y" : "N"); in stmmac_dma_cap_show()
6100 (priv->dma_cap.pcs) ? "Y" : "N"); in stmmac_dma_cap_show()
6102 (priv->dma_cap.sma_mdio) ? "Y" : "N"); in stmmac_dma_cap_show()
6104 (priv->dma_cap.pmt_remote_wake_up) ? "Y" : "N"); in stmmac_dma_cap_show()
6106 (priv->dma_cap.pmt_magic_frame) ? "Y" : "N"); in stmmac_dma_cap_show()
6108 (priv->dma_cap.rmon) ? "Y" : "N"); in stmmac_dma_cap_show()
6110 (priv->dma_cap.time_stamp) ? "Y" : "N"); in stmmac_dma_cap_show()
6112 (priv->dma_cap.atime_stamp) ? "Y" : "N"); in stmmac_dma_cap_show()
6114 (priv->dma_cap.eee) ? "Y" : "N"); in stmmac_dma_cap_show()
6115 seq_printf(seq, "\tAV features: %s\n", (priv->dma_cap.av) ? "Y" : "N"); in stmmac_dma_cap_show()
6117 (priv->dma_cap.tx_coe) ? "Y" : "N"); in stmmac_dma_cap_show()
6118 if (priv->synopsys_id >= DWMAC_CORE_4_00) { in stmmac_dma_cap_show()
6120 (priv->dma_cap.rx_coe) ? "Y" : "N"); in stmmac_dma_cap_show()
6123 (priv->dma_cap.rx_coe_type1) ? "Y" : "N"); in stmmac_dma_cap_show()
6125 (priv->dma_cap.rx_coe_type2) ? "Y" : "N"); in stmmac_dma_cap_show()
6128 (priv->dma_cap.rxfifo_over_2048) ? "Y" : "N"); in stmmac_dma_cap_show()
6130 priv->dma_cap.number_rx_channel); in stmmac_dma_cap_show()
6132 priv->dma_cap.number_tx_channel); in stmmac_dma_cap_show()
6134 priv->dma_cap.number_rx_queues); in stmmac_dma_cap_show()
6136 priv->dma_cap.number_tx_queues); in stmmac_dma_cap_show()
6138 (priv->dma_cap.enh_desc) ? "Y" : "N"); in stmmac_dma_cap_show()
6139 seq_printf(seq, "\tTX Fifo Size: %d\n", priv->dma_cap.tx_fifo_size); in stmmac_dma_cap_show()
6140 seq_printf(seq, "\tRX Fifo Size: %d\n", priv->dma_cap.rx_fifo_size); in stmmac_dma_cap_show()
6141 seq_printf(seq, "\tHash Table Size: %d\n", priv->dma_cap.hash_tb_sz); in stmmac_dma_cap_show()
6142 seq_printf(seq, "\tTSO: %s\n", priv->dma_cap.tsoen ? "Y" : "N"); in stmmac_dma_cap_show()
6144 priv->dma_cap.pps_out_num); in stmmac_dma_cap_show()
6146 priv->dma_cap.asp ? "Y" : "N"); in stmmac_dma_cap_show()
6148 priv->dma_cap.frpsel ? "Y" : "N"); in stmmac_dma_cap_show()
6150 priv->dma_cap.addr64); in stmmac_dma_cap_show()
6152 priv->dma_cap.rssen ? "Y" : "N"); in stmmac_dma_cap_show()
6154 priv->dma_cap.vlhash ? "Y" : "N"); in stmmac_dma_cap_show()
6156 priv->dma_cap.sphen ? "Y" : "N"); in stmmac_dma_cap_show()
6158 priv->dma_cap.vlins ? "Y" : "N"); in stmmac_dma_cap_show()
6160 priv->dma_cap.dvlan ? "Y" : "N"); in stmmac_dma_cap_show()
6162 priv->dma_cap.l3l4fnum); in stmmac_dma_cap_show()
6164 priv->dma_cap.arpoffsel ? "Y" : "N"); in stmmac_dma_cap_show()
6166 priv->dma_cap.estsel ? "Y" : "N"); in stmmac_dma_cap_show()
6168 priv->dma_cap.fpesel ? "Y" : "N"); in stmmac_dma_cap_show()
6170 priv->dma_cap.tbssel ? "Y" : "N"); in stmmac_dma_cap_show()
6181 struct stmmac_priv *priv = netdev_priv(dev); in stmmac_device_event() local
6188 if (priv->dbgfs_dir) in stmmac_device_event()
6189 priv->dbgfs_dir = debugfs_rename(stmmac_fs_dir, in stmmac_device_event()
6190 priv->dbgfs_dir, in stmmac_device_event()
6205 struct stmmac_priv *priv = netdev_priv(dev); in stmmac_init_fs() local
6210 priv->dbgfs_dir = debugfs_create_dir(dev->name, stmmac_fs_dir); in stmmac_init_fs()
6213 debugfs_create_file("descriptors_status", 0444, priv->dbgfs_dir, dev, in stmmac_init_fs()
6217 debugfs_create_file("dma_cap", 0444, priv->dbgfs_dir, dev, in stmmac_init_fs()
6225 struct stmmac_priv *priv = netdev_priv(dev); in stmmac_exit_fs() local
6227 debugfs_remove_recursive(priv->dbgfs_dir); in stmmac_exit_fs()
6255 static int stmmac_vlan_update(struct stmmac_priv *priv, bool is_double) in stmmac_vlan_update() argument
6262 for_each_set_bit(vid, priv->active_vlans, VLAN_N_VID) { in stmmac_vlan_update()
6269 if (!priv->dma_cap.vlhash) { in stmmac_vlan_update()
6277 return stmmac_update_vlan_hash(priv, priv->hw, hash, pmatch, is_double); in stmmac_vlan_update()
6282 struct stmmac_priv *priv = netdev_priv(ndev); in stmmac_vlan_rx_add_vid() local
6286 ret = pm_runtime_resume_and_get(priv->device); in stmmac_vlan_rx_add_vid()
6293 set_bit(vid, priv->active_vlans); in stmmac_vlan_rx_add_vid()
6294 ret = stmmac_vlan_update(priv, is_double); in stmmac_vlan_rx_add_vid()
6296 clear_bit(vid, priv->active_vlans); in stmmac_vlan_rx_add_vid()
6300 if (priv->hw->num_vlan) { in stmmac_vlan_rx_add_vid()
6301 ret = stmmac_add_hw_vlan_rx_fltr(priv, ndev, priv->hw, proto, vid); in stmmac_vlan_rx_add_vid()
6306 pm_runtime_put(priv->device); in stmmac_vlan_rx_add_vid()
6313 struct stmmac_priv *priv = netdev_priv(ndev); in stmmac_vlan_rx_kill_vid() local
6317 ret = pm_runtime_get_sync(priv->device); in stmmac_vlan_rx_kill_vid()
6319 pm_runtime_put_noidle(priv->device); in stmmac_vlan_rx_kill_vid()
6326 clear_bit(vid, priv->active_vlans); in stmmac_vlan_rx_kill_vid()
6328 if (priv->hw->num_vlan) { in stmmac_vlan_rx_kill_vid()
6329 ret = stmmac_del_hw_vlan_rx_fltr(priv, ndev, priv->hw, proto, vid); in stmmac_vlan_rx_kill_vid()
6334 ret = stmmac_vlan_update(priv, is_double); in stmmac_vlan_rx_kill_vid()
6337 pm_runtime_put(priv->device); in stmmac_vlan_rx_kill_vid()
6344 struct stmmac_priv *priv = netdev_priv(dev); in stmmac_bpf() local
6348 return stmmac_xdp_set_prog(priv, bpf->prog, bpf->extack); in stmmac_bpf()
6350 return stmmac_xdp_setup_pool(priv, bpf->xsk.pool, in stmmac_bpf()
6360 struct stmmac_priv *priv = netdev_priv(dev); in stmmac_xdp_xmit() local
6366 if (unlikely(test_bit(STMMAC_DOWN, &priv->state))) in stmmac_xdp_xmit()
6372 queue = stmmac_xdp_get_tx_queue(priv, cpu); in stmmac_xdp_xmit()
6373 nq = netdev_get_tx_queue(priv->dev, queue); in stmmac_xdp_xmit()
6382 res = stmmac_xdp_xmit_xdpf(priv, queue, frames[i], true); in stmmac_xdp_xmit()
6390 stmmac_flush_tx_descriptors(priv, queue); in stmmac_xdp_xmit()
6391 stmmac_tx_timer_arm(priv, queue); in stmmac_xdp_xmit()
6399 void stmmac_disable_rx_queue(struct stmmac_priv *priv, u32 queue) in stmmac_disable_rx_queue() argument
6401 struct stmmac_channel *ch = &priv->channel[queue]; in stmmac_disable_rx_queue()
6405 stmmac_disable_dma_irq(priv, priv->ioaddr, queue, 1, 0); in stmmac_disable_rx_queue()
6408 stmmac_stop_rx_dma(priv, queue); in stmmac_disable_rx_queue()
6409 __free_dma_rx_desc_resources(priv, queue); in stmmac_disable_rx_queue()
6412 void stmmac_enable_rx_queue(struct stmmac_priv *priv, u32 queue) in stmmac_enable_rx_queue() argument
6414 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; in stmmac_enable_rx_queue()
6415 struct stmmac_channel *ch = &priv->channel[queue]; in stmmac_enable_rx_queue()
6420 ret = __alloc_dma_rx_desc_resources(priv, queue); in stmmac_enable_rx_queue()
6422 netdev_err(priv->dev, "Failed to alloc RX desc.\n"); in stmmac_enable_rx_queue()
6426 ret = __init_dma_rx_desc_rings(priv, queue, GFP_KERNEL); in stmmac_enable_rx_queue()
6428 __free_dma_rx_desc_resources(priv, queue); in stmmac_enable_rx_queue()
6429 netdev_err(priv->dev, "Failed to init RX desc.\n"); in stmmac_enable_rx_queue()
6433 stmmac_clear_rx_descriptors(priv, queue); in stmmac_enable_rx_queue()
6435 stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, in stmmac_enable_rx_queue()
6440 stmmac_set_rx_tail_ptr(priv, priv->ioaddr, in stmmac_enable_rx_queue()
6445 stmmac_set_dma_bfsize(priv, priv->ioaddr, in stmmac_enable_rx_queue()
6449 stmmac_set_dma_bfsize(priv, priv->ioaddr, in stmmac_enable_rx_queue()
6450 priv->dma_buf_sz, in stmmac_enable_rx_queue()
6454 stmmac_start_rx_dma(priv, queue); in stmmac_enable_rx_queue()
6457 stmmac_enable_dma_irq(priv, priv->ioaddr, queue, 1, 0); in stmmac_enable_rx_queue()
6461 void stmmac_disable_tx_queue(struct stmmac_priv *priv, u32 queue) in stmmac_disable_tx_queue() argument
6463 struct stmmac_channel *ch = &priv->channel[queue]; in stmmac_disable_tx_queue()
6467 stmmac_disable_dma_irq(priv, priv->ioaddr, queue, 0, 1); in stmmac_disable_tx_queue()
6470 stmmac_stop_tx_dma(priv, queue); in stmmac_disable_tx_queue()
6471 __free_dma_tx_desc_resources(priv, queue); in stmmac_disable_tx_queue()
6474 void stmmac_enable_tx_queue(struct stmmac_priv *priv, u32 queue) in stmmac_enable_tx_queue() argument
6476 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; in stmmac_enable_tx_queue()
6477 struct stmmac_channel *ch = &priv->channel[queue]; in stmmac_enable_tx_queue()
6481 ret = __alloc_dma_tx_desc_resources(priv, queue); in stmmac_enable_tx_queue()
6483 netdev_err(priv->dev, "Failed to alloc TX desc.\n"); in stmmac_enable_tx_queue()
6487 ret = __init_dma_tx_desc_rings(priv, queue); in stmmac_enable_tx_queue()
6489 __free_dma_tx_desc_resources(priv, queue); in stmmac_enable_tx_queue()
6490 netdev_err(priv->dev, "Failed to init TX desc.\n"); in stmmac_enable_tx_queue()
6494 stmmac_clear_tx_descriptors(priv, queue); in stmmac_enable_tx_queue()
6496 stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, in stmmac_enable_tx_queue()
6500 stmmac_enable_tbs(priv, priv->ioaddr, 1, tx_q->queue_index); in stmmac_enable_tx_queue()
6503 stmmac_set_tx_tail_ptr(priv, priv->ioaddr, in stmmac_enable_tx_queue()
6506 stmmac_start_tx_dma(priv, queue); in stmmac_enable_tx_queue()
6509 stmmac_enable_dma_irq(priv, priv->ioaddr, queue, 0, 1); in stmmac_enable_tx_queue()
6515 struct stmmac_priv *priv = netdev_priv(dev); in stmmac_xdp_release() local
6522 stmmac_disable_all_queues(priv); in stmmac_xdp_release()
6524 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) in stmmac_xdp_release()
6525 hrtimer_cancel(&priv->tx_queue[chan].txtimer); in stmmac_xdp_release()
6531 stmmac_stop_all_dma(priv); in stmmac_xdp_release()
6534 free_dma_desc_resources(priv); in stmmac_xdp_release()
6537 stmmac_mac_set(priv, priv->ioaddr, false); in stmmac_xdp_release()
6548 struct stmmac_priv *priv = netdev_priv(dev); in stmmac_xdp_open() local
6549 u32 rx_cnt = priv->plat->rx_queues_to_use; in stmmac_xdp_open()
6550 u32 tx_cnt = priv->plat->tx_queues_to_use; in stmmac_xdp_open()
6559 ret = alloc_dma_desc_resources(priv); in stmmac_xdp_open()
6575 stmmac_init_chan(priv, priv->ioaddr, priv->plat->dma_cfg, chan); in stmmac_xdp_open()
6576 stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 1, 1); in stmmac_xdp_open()
6580 sph_en = (priv->hw->rx_csum > 0) && priv->sph; in stmmac_xdp_open()
6584 rx_q = &priv->rx_queue[chan]; in stmmac_xdp_open()
6586 stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, in stmmac_xdp_open()
6592 stmmac_set_rx_tail_ptr(priv, priv->ioaddr, in stmmac_xdp_open()
6597 stmmac_set_dma_bfsize(priv, priv->ioaddr, in stmmac_xdp_open()
6601 stmmac_set_dma_bfsize(priv, priv->ioaddr, in stmmac_xdp_open()
6602 priv->dma_buf_sz, in stmmac_xdp_open()
6606 stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan); in stmmac_xdp_open()
6611 tx_q = &priv->tx_queue[chan]; in stmmac_xdp_open()
6613 stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, in stmmac_xdp_open()
6617 stmmac_set_tx_tail_ptr(priv, priv->ioaddr, in stmmac_xdp_open()
6625 stmmac_mac_set(priv, priv->ioaddr, true); in stmmac_xdp_open()
6628 stmmac_start_all_dma(priv); in stmmac_xdp_open()
6635 stmmac_enable_all_queues(priv); in stmmac_xdp_open()
6638 stmmac_enable_all_dma_irq(priv); in stmmac_xdp_open()
6643 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) in stmmac_xdp_open()
6644 hrtimer_cancel(&priv->tx_queue[chan].txtimer); in stmmac_xdp_open()
6648 free_dma_desc_resources(priv); in stmmac_xdp_open()
6655 struct stmmac_priv *priv = netdev_priv(dev); in stmmac_xsk_wakeup() local
6660 if (test_bit(STMMAC_DOWN, &priv->state) || in stmmac_xsk_wakeup()
6661 !netif_carrier_ok(priv->dev)) in stmmac_xsk_wakeup()
6664 if (!stmmac_xdp_is_enabled(priv)) in stmmac_xsk_wakeup()
6667 if (queue >= priv->plat->rx_queues_to_use || in stmmac_xsk_wakeup()
6668 queue >= priv->plat->tx_queues_to_use) in stmmac_xsk_wakeup()
6671 rx_q = &priv->rx_queue[queue]; in stmmac_xsk_wakeup()
6672 tx_q = &priv->tx_queue[queue]; in stmmac_xsk_wakeup()
6673 ch = &priv->channel[queue]; in stmmac_xsk_wakeup()
6712 static void stmmac_reset_subtask(struct stmmac_priv *priv) in stmmac_reset_subtask() argument
6714 if (!test_and_clear_bit(STMMAC_RESET_REQUESTED, &priv->state)) in stmmac_reset_subtask()
6716 if (test_bit(STMMAC_DOWN, &priv->state)) in stmmac_reset_subtask()
6719 netdev_err(priv->dev, "Reset adapter.\n"); in stmmac_reset_subtask()
6722 netif_trans_update(priv->dev); in stmmac_reset_subtask()
6723 while (test_and_set_bit(STMMAC_RESETING, &priv->state)) in stmmac_reset_subtask()
6726 set_bit(STMMAC_DOWN, &priv->state); in stmmac_reset_subtask()
6727 dev_close(priv->dev); in stmmac_reset_subtask()
6728 dev_open(priv->dev, NULL); in stmmac_reset_subtask()
6729 clear_bit(STMMAC_DOWN, &priv->state); in stmmac_reset_subtask()
6730 clear_bit(STMMAC_RESETING, &priv->state); in stmmac_reset_subtask()
6736 struct stmmac_priv *priv = container_of(work, struct stmmac_priv, in stmmac_service_task() local
6739 stmmac_reset_subtask(priv); in stmmac_service_task()
6740 clear_bit(STMMAC_SERVICE_SCHED, &priv->state); in stmmac_service_task()
6751 static int stmmac_hw_init(struct stmmac_priv *priv) in stmmac_hw_init() argument
6756 if (priv->plat->has_sun8i) in stmmac_hw_init()
6758 priv->chain_mode = chain_mode; in stmmac_hw_init()
6761 ret = stmmac_hwif_init(priv); in stmmac_hw_init()
6766 priv->hw_cap_support = stmmac_get_hw_features(priv); in stmmac_hw_init()
6767 if (priv->hw_cap_support) { in stmmac_hw_init()
6768 dev_info(priv->device, "DMA HW capability register supported\n"); in stmmac_hw_init()
6775 priv->plat->enh_desc = priv->dma_cap.enh_desc; in stmmac_hw_init()
6776 priv->plat->pmt = priv->dma_cap.pmt_remote_wake_up && in stmmac_hw_init()
6777 !priv->plat->use_phy_wol; in stmmac_hw_init()
6778 priv->hw->pmt = priv->plat->pmt; in stmmac_hw_init()
6779 if (priv->dma_cap.hash_tb_sz) { in stmmac_hw_init()
6780 priv->hw->multicast_filter_bins = in stmmac_hw_init()
6781 (BIT(priv->dma_cap.hash_tb_sz) << 5); in stmmac_hw_init()
6782 priv->hw->mcast_bits_log2 = in stmmac_hw_init()
6783 ilog2(priv->hw->multicast_filter_bins); in stmmac_hw_init()
6787 if (priv->plat->force_thresh_dma_mode) in stmmac_hw_init()
6788 priv->plat->tx_coe = 0; in stmmac_hw_init()
6790 priv->plat->tx_coe = priv->dma_cap.tx_coe; in stmmac_hw_init()
6793 priv->plat->rx_coe = priv->dma_cap.rx_coe; in stmmac_hw_init()
6795 if (priv->dma_cap.rx_coe_type2) in stmmac_hw_init()
6796 priv->plat->rx_coe = STMMAC_RX_COE_TYPE2; in stmmac_hw_init()
6797 else if (priv->dma_cap.rx_coe_type1) in stmmac_hw_init()
6798 priv->plat->rx_coe = STMMAC_RX_COE_TYPE1; in stmmac_hw_init()
6801 dev_info(priv->device, "No HW DMA feature register supported\n"); in stmmac_hw_init()
6804 if (priv->plat->rx_coe) { in stmmac_hw_init()
6805 priv->hw->rx_csum = priv->plat->rx_coe; in stmmac_hw_init()
6806 dev_info(priv->device, "RX Checksum Offload Engine supported\n"); in stmmac_hw_init()
6807 if (priv->synopsys_id < DWMAC_CORE_4_00) in stmmac_hw_init()
6808 dev_info(priv->device, "COE Type %d\n", priv->hw->rx_csum); in stmmac_hw_init()
6810 if (priv->plat->tx_coe) in stmmac_hw_init()
6811 dev_info(priv->device, "TX Checksum insertion supported\n"); in stmmac_hw_init()
6813 if (priv->plat->pmt) { in stmmac_hw_init()
6814 dev_info(priv->device, "Wake-Up On Lan supported\n"); in stmmac_hw_init()
6815 device_set_wakeup_capable(priv->device, 1); in stmmac_hw_init()
6818 if (priv->dma_cap.tsoen) in stmmac_hw_init()
6819 dev_info(priv->device, "TSO supported\n"); in stmmac_hw_init()
6821 priv->hw->vlan_fail_q_en = priv->plat->vlan_fail_q_en; in stmmac_hw_init()
6822 priv->hw->vlan_fail_q = priv->plat->vlan_fail_q; in stmmac_hw_init()
6825 if (priv->hwif_quirks) { in stmmac_hw_init()
6826 ret = priv->hwif_quirks(priv); in stmmac_hw_init()
6836 if (((priv->synopsys_id >= DWMAC_CORE_3_50) || in stmmac_hw_init()
6837 (priv->plat->has_xgmac)) && (!priv->plat->riwt_off)) { in stmmac_hw_init()
6838 priv->use_riwt = 1; in stmmac_hw_init()
6839 dev_info(priv->device, in stmmac_hw_init()
6848 struct stmmac_priv *priv = netdev_priv(dev); in stmmac_napi_add() local
6851 maxq = max(priv->plat->rx_queues_to_use, priv->plat->tx_queues_to_use); in stmmac_napi_add()
6854 struct stmmac_channel *ch = &priv->channel[queue]; in stmmac_napi_add()
6856 ch->priv_data = priv; in stmmac_napi_add()
6860 if (queue < priv->plat->rx_queues_to_use) { in stmmac_napi_add()
6864 if (queue < priv->plat->tx_queues_to_use) { in stmmac_napi_add()
6869 if (queue < priv->plat->rx_queues_to_use && in stmmac_napi_add()
6870 queue < priv->plat->tx_queues_to_use) { in stmmac_napi_add()
6880 struct stmmac_priv *priv = netdev_priv(dev); in stmmac_napi_del() local
6883 maxq = max(priv->plat->rx_queues_to_use, priv->plat->tx_queues_to_use); in stmmac_napi_del()
6886 struct stmmac_channel *ch = &priv->channel[queue]; in stmmac_napi_del()
6888 if (queue < priv->plat->rx_queues_to_use) in stmmac_napi_del()
6890 if (queue < priv->plat->tx_queues_to_use) in stmmac_napi_del()
6892 if (queue < priv->plat->rx_queues_to_use && in stmmac_napi_del()
6893 queue < priv->plat->tx_queues_to_use) { in stmmac_napi_del()
6901 struct stmmac_priv *priv = netdev_priv(dev); in stmmac_reinit_queues() local
6909 priv->plat->rx_queues_to_use = rx_cnt; in stmmac_reinit_queues()
6910 priv->plat->tx_queues_to_use = tx_cnt; in stmmac_reinit_queues()
6912 for (i = 0; i < ARRAY_SIZE(priv->rss.table); i++) in stmmac_reinit_queues()
6913 priv->rss.table[i] = ethtool_rxfh_indir_default(i, in stmmac_reinit_queues()
6926 struct stmmac_priv *priv = netdev_priv(dev); in stmmac_reinit_ringparam() local
6932 priv->dma_rx_size = rx_size; in stmmac_reinit_ringparam()
6933 priv->dma_tx_size = tx_size; in stmmac_reinit_ringparam()
6944 struct stmmac_priv *priv = container_of(work, struct stmmac_priv, in stmmac_fpe_lp_task() local
6946 struct stmmac_fpe_cfg *fpe_cfg = priv->plat->fpe_cfg; in stmmac_fpe_lp_task()
6960 stmmac_fpe_configure(priv, priv->ioaddr, in stmmac_fpe_lp_task()
6962 priv->plat->tx_queues_to_use, in stmmac_fpe_lp_task()
6963 priv->plat->rx_queues_to_use, in stmmac_fpe_lp_task()
6966 netdev_info(priv->dev, "configured FPE\n"); in stmmac_fpe_lp_task()
6970 netdev_info(priv->dev, "!!! BOTH FPE stations ON\n"); in stmmac_fpe_lp_task()
6977 netdev_info(priv->dev, SEND_VERIFY_MPAKCET_FMT, in stmmac_fpe_lp_task()
6979 stmmac_fpe_send_mpacket(priv, priv->ioaddr, in stmmac_fpe_lp_task()
6987 clear_bit(__FPE_TASK_SCHED, &priv->fpe_task_state); in stmmac_fpe_lp_task()
6990 void stmmac_fpe_handshake(struct stmmac_priv *priv, bool enable) in stmmac_fpe_handshake() argument
6992 if (priv->plat->fpe_cfg->hs_enable != enable) { in stmmac_fpe_handshake()
6994 stmmac_fpe_send_mpacket(priv, priv->ioaddr, in stmmac_fpe_handshake()
6995 priv->plat->fpe_cfg, in stmmac_fpe_handshake()
6998 priv->plat->fpe_cfg->lo_fpe_state = FPE_STATE_OFF; in stmmac_fpe_handshake()
6999 priv->plat->fpe_cfg->lp_fpe_state = FPE_STATE_OFF; in stmmac_fpe_handshake()
7002 priv->plat->fpe_cfg->hs_enable = enable; in stmmac_fpe_handshake()
7021 struct stmmac_priv *priv; in stmmac_dvr_probe() local
7032 priv = netdev_priv(ndev); in stmmac_dvr_probe()
7033 priv->device = device; in stmmac_dvr_probe()
7034 priv->dev = ndev; in stmmac_dvr_probe()
7037 priv->pause = pause; in stmmac_dvr_probe()
7038 priv->plat = plat_dat; in stmmac_dvr_probe()
7039 priv->ioaddr = res->addr; in stmmac_dvr_probe()
7040 priv->dev->base_addr = (unsigned long)res->addr; in stmmac_dvr_probe()
7041 priv->plat->dma_cfg->multi_msi_en = priv->plat->multi_msi_en; in stmmac_dvr_probe()
7043 priv->dev->irq = res->irq; in stmmac_dvr_probe()
7044 priv->wol_irq = res->wol_irq; in stmmac_dvr_probe()
7045 priv->lpi_irq = res->lpi_irq; in stmmac_dvr_probe()
7046 priv->sfty_ce_irq = res->sfty_ce_irq; in stmmac_dvr_probe()
7047 priv->sfty_ue_irq = res->sfty_ue_irq; in stmmac_dvr_probe()
7049 priv->rx_irq[i] = res->rx_irq[i]; in stmmac_dvr_probe()
7051 priv->tx_irq[i] = res->tx_irq[i]; in stmmac_dvr_probe()
7054 memcpy(priv->dev->dev_addr, res->mac, ETH_ALEN); in stmmac_dvr_probe()
7056 dev_set_drvdata(device, priv->dev); in stmmac_dvr_probe()
7061 priv->af_xdp_zc_qps = bitmap_zalloc(MTL_MAX_TX_QUEUES, GFP_KERNEL); in stmmac_dvr_probe()
7062 if (!priv->af_xdp_zc_qps) in stmmac_dvr_probe()
7066 priv->wq = create_singlethread_workqueue("stmmac_wq"); in stmmac_dvr_probe()
7067 if (!priv->wq) { in stmmac_dvr_probe()
7068 dev_err(priv->device, "failed to create workqueue\n"); in stmmac_dvr_probe()
7073 INIT_WORK(&priv->service_task, stmmac_service_task); in stmmac_dvr_probe()
7076 INIT_WORK(&priv->fpe_task, stmmac_fpe_lp_task); in stmmac_dvr_probe()
7082 priv->plat->phy_addr = phyaddr; in stmmac_dvr_probe()
7084 if (priv->plat->stmmac_rst) { in stmmac_dvr_probe()
7085 ret = reset_control_assert(priv->plat->stmmac_rst); in stmmac_dvr_probe()
7086 reset_control_deassert(priv->plat->stmmac_rst); in stmmac_dvr_probe()
7091 reset_control_reset(priv->plat->stmmac_rst); in stmmac_dvr_probe()
7094 ret = reset_control_deassert(priv->plat->stmmac_ahb_rst); in stmmac_dvr_probe()
7096 dev_err(priv->device, "unable to bring out of ahb reset: %pe\n", in stmmac_dvr_probe()
7103 ret = stmmac_hw_init(priv); in stmmac_dvr_probe()
7109 if (priv->synopsys_id < DWMAC_CORE_5_20) in stmmac_dvr_probe()
7110 priv->plat->dma_cfg->dche = false; in stmmac_dvr_probe()
7112 stmmac_check_ether_addr(priv); in stmmac_dvr_probe()
7119 ret = stmmac_tc_init(priv, priv); in stmmac_dvr_probe()
7124 if ((priv->plat->tso_en) && (priv->dma_cap.tsoen)) { in stmmac_dvr_probe()
7126 if (priv->plat->has_gmac4) in stmmac_dvr_probe()
7128 priv->tso = true; in stmmac_dvr_probe()
7129 dev_info(priv->device, "TSO feature enabled\n"); in stmmac_dvr_probe()
7132 if (priv->dma_cap.sphen && !priv->plat->sph_disable) { in stmmac_dvr_probe()
7134 priv->sph_cap = true; in stmmac_dvr_probe()
7135 priv->sph = priv->sph_cap; in stmmac_dvr_probe()
7136 dev_info(priv->device, "SPH feature enabled\n"); in stmmac_dvr_probe()
7144 if (priv->plat->addr64) in stmmac_dvr_probe()
7145 priv->dma_cap.addr64 = priv->plat->addr64; in stmmac_dvr_probe()
7147 if (priv->dma_cap.addr64) { in stmmac_dvr_probe()
7149 DMA_BIT_MASK(priv->dma_cap.addr64)); in stmmac_dvr_probe()
7151 dev_info(priv->device, "Using %d bits DMA width\n", in stmmac_dvr_probe()
7152 priv->dma_cap.addr64); in stmmac_dvr_probe()
7159 priv->plat->dma_cfg->eame = true; in stmmac_dvr_probe()
7163 dev_err(priv->device, "Failed to set DMA Mask\n"); in stmmac_dvr_probe()
7167 priv->dma_cap.addr64 = 32; in stmmac_dvr_probe()
7176 if (priv->dma_cap.vlhash) { in stmmac_dvr_probe()
7180 if (priv->dma_cap.vlins) { in stmmac_dvr_probe()
7182 if (priv->dma_cap.dvlan) in stmmac_dvr_probe()
7186 priv->msg_enable = netif_msg_init(debug, default_msg_level); in stmmac_dvr_probe()
7189 rxq = priv->plat->rx_queues_to_use; in stmmac_dvr_probe()
7190 netdev_rss_key_fill(priv->rss.key, sizeof(priv->rss.key)); in stmmac_dvr_probe()
7191 for (i = 0; i < ARRAY_SIZE(priv->rss.table); i++) in stmmac_dvr_probe()
7192 priv->rss.table[i] = ethtool_rxfh_indir_default(i, rxq); in stmmac_dvr_probe()
7194 if (priv->dma_cap.rssen && priv->plat->rss_en) in stmmac_dvr_probe()
7199 if (priv->plat->has_xgmac) in stmmac_dvr_probe()
7201 else if ((priv->plat->enh_desc) || (priv->synopsys_id >= DWMAC_CORE_4_00)) in stmmac_dvr_probe()
7208 if ((priv->plat->maxmtu < ndev->max_mtu) && in stmmac_dvr_probe()
7209 (priv->plat->maxmtu >= ndev->min_mtu)) in stmmac_dvr_probe()
7210 ndev->max_mtu = priv->plat->maxmtu; in stmmac_dvr_probe()
7211 else if (priv->plat->maxmtu < ndev->min_mtu) in stmmac_dvr_probe()
7212 dev_warn(priv->device, in stmmac_dvr_probe()
7214 __func__, priv->plat->maxmtu); in stmmac_dvr_probe()
7217 priv->flow_ctrl = FLOW_AUTO; /* RX/TX pause on */ in stmmac_dvr_probe()
7222 mutex_init(&priv->lock); in stmmac_dvr_probe()
7230 if (priv->plat->clk_csr >= 0) in stmmac_dvr_probe()
7231 priv->clk_csr = priv->plat->clk_csr; in stmmac_dvr_probe()
7233 stmmac_clk_csr_set(priv); in stmmac_dvr_probe()
7235 stmmac_check_pcs_mode(priv); in stmmac_dvr_probe()
7241 if (priv->hw->pcs != STMMAC_PCS_TBI && in stmmac_dvr_probe()
7242 priv->hw->pcs != STMMAC_PCS_RTBI) { in stmmac_dvr_probe()
7246 dev_err_probe(priv->device, ret, in stmmac_dvr_probe()
7248 __func__, priv->plat->bus_id); in stmmac_dvr_probe()
7253 if (priv->plat->speed_mode_2500) in stmmac_dvr_probe()
7254 priv->plat->speed_mode_2500(ndev, priv->plat->bsp_priv); in stmmac_dvr_probe()
7256 if (priv->plat->mdio_bus_data && priv->plat->mdio_bus_data->has_xpcs) { in stmmac_dvr_probe()
7257 ret = stmmac_xpcs_setup(priv->mii); in stmmac_dvr_probe()
7262 ret = stmmac_phy_setup(priv); in stmmac_dvr_probe()
7270 dev_err(priv->device, "%s: ERROR %i registering the device\n", in stmmac_dvr_probe()
7279 if (priv->plat->dump_debug_regs) in stmmac_dvr_probe()
7280 priv->plat->dump_debug_regs(priv->plat->bsp_priv); in stmmac_dvr_probe()
7290 phylink_destroy(priv->phylink); in stmmac_dvr_probe()
7293 if (priv->hw->pcs != STMMAC_PCS_TBI && in stmmac_dvr_probe()
7294 priv->hw->pcs != STMMAC_PCS_RTBI) in stmmac_dvr_probe()
7299 destroy_workqueue(priv->wq); in stmmac_dvr_probe()
7301 bitmap_free(priv->af_xdp_zc_qps); in stmmac_dvr_probe()
7316 struct stmmac_priv *priv = netdev_priv(ndev); in stmmac_dvr_remove() local
7318 netdev_info(priv->dev, "%s: removing driver", __func__); in stmmac_dvr_remove()
7322 stmmac_stop_all_dma(priv); in stmmac_dvr_remove()
7323 stmmac_mac_set(priv, priv->ioaddr, false); in stmmac_dvr_remove()
7330 phylink_destroy(priv->phylink); in stmmac_dvr_remove()
7331 if (priv->plat->stmmac_rst) in stmmac_dvr_remove()
7332 reset_control_assert(priv->plat->stmmac_rst); in stmmac_dvr_remove()
7333 reset_control_assert(priv->plat->stmmac_ahb_rst); in stmmac_dvr_remove()
7334 if (priv->hw->pcs != STMMAC_PCS_TBI && in stmmac_dvr_remove()
7335 priv->hw->pcs != STMMAC_PCS_RTBI) in stmmac_dvr_remove()
7337 destroy_workqueue(priv->wq); in stmmac_dvr_remove()
7338 mutex_destroy(&priv->lock); in stmmac_dvr_remove()
7339 bitmap_free(priv->af_xdp_zc_qps); in stmmac_dvr_remove()
7358 struct stmmac_priv *priv = netdev_priv(ndev); in stmmac_suspend() local
7364 mutex_lock(&priv->lock); in stmmac_suspend()
7368 stmmac_disable_all_queues(priv); in stmmac_suspend()
7370 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) in stmmac_suspend()
7371 hrtimer_cancel(&priv->tx_queue[chan].txtimer); in stmmac_suspend()
7373 if (priv->eee_enabled) { in stmmac_suspend()
7374 priv->tx_path_in_lpi_mode = false; in stmmac_suspend()
7375 del_timer_sync(&priv->eee_ctrl_timer); in stmmac_suspend()
7379 stmmac_stop_all_dma(priv); in stmmac_suspend()
7381 if (priv->plat->serdes_powerdown) in stmmac_suspend()
7382 priv->plat->serdes_powerdown(ndev, priv->plat->bsp_priv); in stmmac_suspend()
7385 if (device_may_wakeup(priv->device) && priv->plat->pmt) { in stmmac_suspend()
7386 stmmac_pmt(priv, priv->hw, priv->wolopts); in stmmac_suspend()
7387 priv->irq_wake = 1; in stmmac_suspend()
7389 stmmac_mac_set(priv, priv->ioaddr, false); in stmmac_suspend()
7390 pinctrl_pm_select_sleep_state(priv->device); in stmmac_suspend()
7393 mutex_unlock(&priv->lock); in stmmac_suspend()
7396 if (device_may_wakeup(priv->device) && priv->plat->pmt) { in stmmac_suspend()
7397 phylink_suspend(priv->phylink, true); in stmmac_suspend()
7399 if (device_may_wakeup(priv->device)) in stmmac_suspend()
7400 phylink_speed_down(priv->phylink, false); in stmmac_suspend()
7401 phylink_suspend(priv->phylink, false); in stmmac_suspend()
7405 if (priv->dma_cap.fpesel) { in stmmac_suspend()
7407 stmmac_fpe_configure(priv, priv->ioaddr, in stmmac_suspend()
7408 priv->plat->fpe_cfg, in stmmac_suspend()
7409 priv->plat->tx_queues_to_use, in stmmac_suspend()
7410 priv->plat->rx_queues_to_use, false); in stmmac_suspend()
7412 stmmac_fpe_handshake(priv, false); in stmmac_suspend()
7413 stmmac_fpe_stop_wq(priv); in stmmac_suspend()
7416 priv->speed = SPEED_UNKNOWN; in stmmac_suspend()
7425 static void stmmac_reset_queues_param(struct stmmac_priv *priv) in stmmac_reset_queues_param() argument
7427 u32 rx_cnt = priv->plat->rx_queues_to_use; in stmmac_reset_queues_param()
7428 u32 tx_cnt = priv->plat->tx_queues_to_use; in stmmac_reset_queues_param()
7432 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; in stmmac_reset_queues_param()
7439 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; in stmmac_reset_queues_param()
7445 netdev_tx_reset_queue(netdev_get_tx_queue(priv->dev, queue)); in stmmac_reset_queues_param()
7458 struct stmmac_priv *priv = netdev_priv(ndev); in stmmac_resume() local
7470 if (device_may_wakeup(priv->device) && priv->plat->pmt) { in stmmac_resume()
7471 mutex_lock(&priv->lock); in stmmac_resume()
7472 stmmac_pmt(priv, priv->hw, 0); in stmmac_resume()
7473 mutex_unlock(&priv->lock); in stmmac_resume()
7474 priv->irq_wake = 0; in stmmac_resume()
7476 pinctrl_pm_select_default_state(priv->device); in stmmac_resume()
7478 if (priv->mii) in stmmac_resume()
7479 stmmac_mdio_reset(priv->mii); in stmmac_resume()
7482 if (priv->plat->serdes_powerup) { in stmmac_resume()
7483 ret = priv->plat->serdes_powerup(ndev, in stmmac_resume()
7484 priv->plat->bsp_priv); in stmmac_resume()
7491 if (device_may_wakeup(priv->device) && priv->plat->pmt) { in stmmac_resume()
7492 phylink_resume(priv->phylink); in stmmac_resume()
7494 phylink_resume(priv->phylink); in stmmac_resume()
7495 if (device_may_wakeup(priv->device)) in stmmac_resume()
7496 phylink_speed_up(priv->phylink); in stmmac_resume()
7501 mutex_lock(&priv->lock); in stmmac_resume()
7503 stmmac_reset_queues_param(priv); in stmmac_resume()
7505 stmmac_free_tx_skbufs(priv); in stmmac_resume()
7506 stmmac_clear_descriptors(priv); in stmmac_resume()
7509 stmmac_init_coalesce(priv); in stmmac_resume()
7512 stmmac_restore_hw_vlan_rx_fltr(priv, ndev, priv->hw); in stmmac_resume()
7514 stmmac_enable_all_queues(priv); in stmmac_resume()
7515 stmmac_enable_all_dma_irq(priv); in stmmac_resume()
7517 mutex_unlock(&priv->lock); in stmmac_resume()