1 /*
2 * aQuantia Corporation Network Driver
3 * Copyright (C) 2014-2017 aQuantia Corporation. All rights reserved
4 *
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
8 */
9
10 /* File aq_vec.c: Definition of common structure for vector of Rx and Tx rings.
11 * Definition of functions for Rx and Tx rings. Friendly module for aq_nic.
12 */
13
14 #include "aq_vec.h"
15 #include "aq_nic.h"
16 #include "aq_ring.h"
17 #include "aq_hw.h"
18
19 #include <linux/netdevice.h>
20
21 struct aq_vec_s {
22 const struct aq_hw_ops *aq_hw_ops;
23 struct aq_hw_s *aq_hw;
24 struct aq_nic_s *aq_nic;
25 unsigned int tx_rings;
26 unsigned int rx_rings;
27 struct aq_ring_param_s aq_ring_param;
28 struct napi_struct napi;
29 struct aq_ring_s ring[AQ_CFG_TCS_MAX][2];
30 };
31
32 #define AQ_VEC_TX_ID 0
33 #define AQ_VEC_RX_ID 1
34
aq_vec_poll(struct napi_struct * napi,int budget)35 static int aq_vec_poll(struct napi_struct *napi, int budget)
36 {
37 struct aq_vec_s *self = container_of(napi, struct aq_vec_s, napi);
38 unsigned int sw_tail_old = 0U;
39 struct aq_ring_s *ring = NULL;
40 bool was_tx_cleaned = true;
41 unsigned int i = 0U;
42 int work_done = 0;
43 int err = 0;
44
45 if (!self) {
46 err = -EINVAL;
47 } else {
48 for (i = 0U, ring = self->ring[0];
49 self->tx_rings > i; ++i, ring = self->ring[i]) {
50 if (self->aq_hw_ops->hw_ring_tx_head_update) {
51 err = self->aq_hw_ops->hw_ring_tx_head_update(
52 self->aq_hw,
53 &ring[AQ_VEC_TX_ID]);
54 if (err < 0)
55 goto err_exit;
56 }
57
58 if (ring[AQ_VEC_TX_ID].sw_head !=
59 ring[AQ_VEC_TX_ID].hw_head) {
60 was_tx_cleaned = aq_ring_tx_clean(&ring[AQ_VEC_TX_ID]);
61 aq_ring_update_queue_state(&ring[AQ_VEC_TX_ID]);
62 }
63
64 err = self->aq_hw_ops->hw_ring_rx_receive(self->aq_hw,
65 &ring[AQ_VEC_RX_ID]);
66 if (err < 0)
67 goto err_exit;
68
69 if (ring[AQ_VEC_RX_ID].sw_head !=
70 ring[AQ_VEC_RX_ID].hw_head) {
71 err = aq_ring_rx_clean(&ring[AQ_VEC_RX_ID],
72 napi,
73 &work_done,
74 budget - work_done);
75 if (err < 0)
76 goto err_exit;
77
78 sw_tail_old = ring[AQ_VEC_RX_ID].sw_tail;
79
80 err = aq_ring_rx_fill(&ring[AQ_VEC_RX_ID]);
81 if (err < 0)
82 goto err_exit;
83
84 err = self->aq_hw_ops->hw_ring_rx_fill(
85 self->aq_hw,
86 &ring[AQ_VEC_RX_ID], sw_tail_old);
87 if (err < 0)
88 goto err_exit;
89 }
90 }
91
92 err_exit:
93 if (!was_tx_cleaned)
94 work_done = budget;
95
96 if (work_done < budget) {
97 napi_complete_done(napi, work_done);
98 self->aq_hw_ops->hw_irq_enable(self->aq_hw,
99 1U << self->aq_ring_param.vec_idx);
100 }
101 }
102
103 return work_done;
104 }
105
aq_vec_alloc(struct aq_nic_s * aq_nic,unsigned int idx,struct aq_nic_cfg_s * aq_nic_cfg)106 struct aq_vec_s *aq_vec_alloc(struct aq_nic_s *aq_nic, unsigned int idx,
107 struct aq_nic_cfg_s *aq_nic_cfg)
108 {
109 struct aq_vec_s *self = NULL;
110 struct aq_ring_s *ring = NULL;
111 unsigned int i = 0U;
112 int err = 0;
113
114 self = kzalloc(sizeof(*self), GFP_KERNEL);
115 if (!self) {
116 err = -ENOMEM;
117 goto err_exit;
118 }
119
120 self->aq_nic = aq_nic;
121 self->aq_ring_param.vec_idx = idx;
122 self->aq_ring_param.cpu =
123 idx + aq_nic_cfg->aq_rss.base_cpu_number;
124
125 cpumask_set_cpu(self->aq_ring_param.cpu,
126 &self->aq_ring_param.affinity_mask);
127
128 self->tx_rings = 0;
129 self->rx_rings = 0;
130
131 netif_napi_add(aq_nic_get_ndev(aq_nic), &self->napi,
132 aq_vec_poll, AQ_CFG_NAPI_WEIGHT);
133
134 for (i = 0; i < aq_nic_cfg->tcs; ++i) {
135 unsigned int idx_ring = AQ_NIC_TCVEC2RING(self->nic,
136 self->tx_rings,
137 self->aq_ring_param.vec_idx);
138
139 ring = aq_ring_tx_alloc(&self->ring[i][AQ_VEC_TX_ID], aq_nic,
140 idx_ring, aq_nic_cfg);
141 if (!ring) {
142 err = -ENOMEM;
143 goto err_exit;
144 }
145
146 ++self->tx_rings;
147
148 aq_nic_set_tx_ring(aq_nic, idx_ring, ring);
149
150 ring = aq_ring_rx_alloc(&self->ring[i][AQ_VEC_RX_ID], aq_nic,
151 idx_ring, aq_nic_cfg);
152 if (!ring) {
153 err = -ENOMEM;
154 goto err_exit;
155 }
156
157 ++self->rx_rings;
158 }
159
160 err_exit:
161 if (err < 0) {
162 aq_vec_free(self);
163 self = NULL;
164 }
165 return self;
166 }
167
aq_vec_init(struct aq_vec_s * self,const struct aq_hw_ops * aq_hw_ops,struct aq_hw_s * aq_hw)168 int aq_vec_init(struct aq_vec_s *self, const struct aq_hw_ops *aq_hw_ops,
169 struct aq_hw_s *aq_hw)
170 {
171 struct aq_ring_s *ring = NULL;
172 unsigned int i = 0U;
173 int err = 0;
174
175 self->aq_hw_ops = aq_hw_ops;
176 self->aq_hw = aq_hw;
177
178 for (i = 0U, ring = self->ring[0];
179 self->tx_rings > i; ++i, ring = self->ring[i]) {
180 err = aq_ring_init(&ring[AQ_VEC_TX_ID]);
181 if (err < 0)
182 goto err_exit;
183
184 err = self->aq_hw_ops->hw_ring_tx_init(self->aq_hw,
185 &ring[AQ_VEC_TX_ID],
186 &self->aq_ring_param);
187 if (err < 0)
188 goto err_exit;
189
190 err = aq_ring_init(&ring[AQ_VEC_RX_ID]);
191 if (err < 0)
192 goto err_exit;
193
194 err = self->aq_hw_ops->hw_ring_rx_init(self->aq_hw,
195 &ring[AQ_VEC_RX_ID],
196 &self->aq_ring_param);
197 if (err < 0)
198 goto err_exit;
199
200 err = aq_ring_rx_fill(&ring[AQ_VEC_RX_ID]);
201 if (err < 0)
202 goto err_exit;
203
204 err = self->aq_hw_ops->hw_ring_rx_fill(self->aq_hw,
205 &ring[AQ_VEC_RX_ID], 0U);
206 if (err < 0)
207 goto err_exit;
208 }
209
210 err_exit:
211 return err;
212 }
213
aq_vec_start(struct aq_vec_s * self)214 int aq_vec_start(struct aq_vec_s *self)
215 {
216 struct aq_ring_s *ring = NULL;
217 unsigned int i = 0U;
218 int err = 0;
219
220 for (i = 0U, ring = self->ring[0];
221 self->tx_rings > i; ++i, ring = self->ring[i]) {
222 err = self->aq_hw_ops->hw_ring_tx_start(self->aq_hw,
223 &ring[AQ_VEC_TX_ID]);
224 if (err < 0)
225 goto err_exit;
226
227 err = self->aq_hw_ops->hw_ring_rx_start(self->aq_hw,
228 &ring[AQ_VEC_RX_ID]);
229 if (err < 0)
230 goto err_exit;
231 }
232
233 napi_enable(&self->napi);
234
235 err_exit:
236 return err;
237 }
238
aq_vec_stop(struct aq_vec_s * self)239 void aq_vec_stop(struct aq_vec_s *self)
240 {
241 struct aq_ring_s *ring = NULL;
242 unsigned int i = 0U;
243
244 for (i = 0U, ring = self->ring[0];
245 self->tx_rings > i; ++i, ring = self->ring[i]) {
246 self->aq_hw_ops->hw_ring_tx_stop(self->aq_hw,
247 &ring[AQ_VEC_TX_ID]);
248
249 self->aq_hw_ops->hw_ring_rx_stop(self->aq_hw,
250 &ring[AQ_VEC_RX_ID]);
251 }
252
253 napi_disable(&self->napi);
254 }
255
aq_vec_deinit(struct aq_vec_s * self)256 void aq_vec_deinit(struct aq_vec_s *self)
257 {
258 struct aq_ring_s *ring = NULL;
259 unsigned int i = 0U;
260
261 if (!self)
262 goto err_exit;
263
264 for (i = 0U, ring = self->ring[0];
265 self->tx_rings > i; ++i, ring = self->ring[i]) {
266 aq_ring_tx_clean(&ring[AQ_VEC_TX_ID]);
267 aq_ring_rx_deinit(&ring[AQ_VEC_RX_ID]);
268 }
269 err_exit:;
270 }
271
aq_vec_free(struct aq_vec_s * self)272 void aq_vec_free(struct aq_vec_s *self)
273 {
274 struct aq_ring_s *ring = NULL;
275 unsigned int i = 0U;
276
277 if (!self)
278 goto err_exit;
279
280 for (i = 0U, ring = self->ring[0];
281 self->tx_rings > i; ++i, ring = self->ring[i]) {
282 aq_ring_free(&ring[AQ_VEC_TX_ID]);
283 aq_ring_free(&ring[AQ_VEC_RX_ID]);
284 }
285
286 netif_napi_del(&self->napi);
287
288 kfree(self);
289
290 err_exit:;
291 }
292
aq_vec_isr(int irq,void * private)293 irqreturn_t aq_vec_isr(int irq, void *private)
294 {
295 struct aq_vec_s *self = private;
296 int err = 0;
297
298 if (!self) {
299 err = -EINVAL;
300 goto err_exit;
301 }
302 napi_schedule(&self->napi);
303
304 err_exit:
305 return err >= 0 ? IRQ_HANDLED : IRQ_NONE;
306 }
307
aq_vec_isr_legacy(int irq,void * private)308 irqreturn_t aq_vec_isr_legacy(int irq, void *private)
309 {
310 struct aq_vec_s *self = private;
311 u64 irq_mask = 0U;
312 int err;
313
314 if (!self)
315 return IRQ_NONE;
316 err = self->aq_hw_ops->hw_irq_read(self->aq_hw, &irq_mask);
317 if (err < 0)
318 return IRQ_NONE;
319
320 if (irq_mask) {
321 self->aq_hw_ops->hw_irq_disable(self->aq_hw,
322 1U << self->aq_ring_param.vec_idx);
323 napi_schedule(&self->napi);
324 } else {
325 self->aq_hw_ops->hw_irq_enable(self->aq_hw, 1U);
326 return IRQ_NONE;
327 }
328
329 return IRQ_HANDLED;
330 }
331
aq_vec_get_affinity_mask(struct aq_vec_s * self)332 cpumask_t *aq_vec_get_affinity_mask(struct aq_vec_s *self)
333 {
334 return &self->aq_ring_param.affinity_mask;
335 }
336
aq_vec_add_stats(struct aq_vec_s * self,struct aq_ring_stats_rx_s * stats_rx,struct aq_ring_stats_tx_s * stats_tx)337 void aq_vec_add_stats(struct aq_vec_s *self,
338 struct aq_ring_stats_rx_s *stats_rx,
339 struct aq_ring_stats_tx_s *stats_tx)
340 {
341 struct aq_ring_s *ring = NULL;
342 unsigned int r = 0U;
343
344 for (r = 0U, ring = self->ring[0];
345 self->tx_rings > r; ++r, ring = self->ring[r]) {
346 struct aq_ring_stats_tx_s *tx = &ring[AQ_VEC_TX_ID].stats.tx;
347 struct aq_ring_stats_rx_s *rx = &ring[AQ_VEC_RX_ID].stats.rx;
348
349 stats_rx->packets += rx->packets;
350 stats_rx->bytes += rx->bytes;
351 stats_rx->errors += rx->errors;
352 stats_rx->jumbo_packets += rx->jumbo_packets;
353 stats_rx->lro_packets += rx->lro_packets;
354
355 stats_tx->packets += tx->packets;
356 stats_tx->bytes += tx->bytes;
357 stats_tx->errors += tx->errors;
358 stats_tx->queue_restarts += tx->queue_restarts;
359 }
360 }
361
aq_vec_get_sw_stats(struct aq_vec_s * self,u64 * data,unsigned int * p_count)362 int aq_vec_get_sw_stats(struct aq_vec_s *self, u64 *data, unsigned int *p_count)
363 {
364 unsigned int count = 0U;
365 struct aq_ring_stats_rx_s stats_rx;
366 struct aq_ring_stats_tx_s stats_tx;
367
368 memset(&stats_rx, 0U, sizeof(struct aq_ring_stats_rx_s));
369 memset(&stats_tx, 0U, sizeof(struct aq_ring_stats_tx_s));
370 aq_vec_add_stats(self, &stats_rx, &stats_tx);
371
372 /* This data should mimic aq_ethtool_queue_stat_names structure
373 */
374 data[count] += stats_rx.packets;
375 data[++count] += stats_tx.packets;
376 data[++count] += stats_tx.queue_restarts;
377 data[++count] += stats_rx.jumbo_packets;
378 data[++count] += stats_rx.lro_packets;
379 data[++count] += stats_rx.errors;
380
381 if (p_count)
382 *p_count = ++count;
383
384 return 0;
385 }
386