• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /**************************************************************************/
2 /*                                                                        */
3 /*  IBM System i and System p Virtual NIC Device Driver                   */
4 /*  Copyright (C) 2014 IBM Corp.                                          */
5 /*  Santiago Leon (santi_leon@yahoo.com)                                  */
6 /*  Thomas Falcon (tlfalcon@linux.vnet.ibm.com)                           */
7 /*  John Allen (jallen@linux.vnet.ibm.com)                                */
8 /*                                                                        */
9 /*  This program is free software; you can redistribute it and/or modify  */
10 /*  it under the terms of the GNU General Public License as published by  */
11 /*  the Free Software Foundation; either version 2 of the License, or     */
12 /*  (at your option) any later version.                                   */
13 /*                                                                        */
14 /*  This program is distributed in the hope that it will be useful,       */
15 /*  but WITHOUT ANY WARRANTY; without even the implied warranty of        */
16 /*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         */
17 /*  GNU General Public License for more details.                          */
18 /*                                                                        */
19 /*  You should have received a copy of the GNU General Public License     */
20 /*  along with this program.                                              */
21 /*                                                                        */
22 /* This module contains the implementation of a virtual ethernet device   */
23 /* for use with IBM i/p Series LPAR Linux. It utilizes the logical LAN    */
24 /* option of the RS/6000 Platform Architecture to interface with virtual  */
25 /* ethernet NICs that are presented to the partition by the hypervisor.   */
26 /*									   */
27 /* Messages are passed between the VNIC driver and the VNIC server using  */
28 /* Command/Response Queues (CRQs) and sub CRQs (sCRQs). CRQs are used to  */
29 /* issue and receive commands that initiate communication with the server */
30 /* on driver initialization. Sub CRQs (sCRQs) are similar to CRQs, but    */
31 /* are used by the driver to notify the server that a packet is           */
32 /* ready for transmission or that a buffer has been added to receive a    */
33 /* packet. Subsequently, sCRQs are used by the server to notify the       */
34 /* driver that a packet transmission has been completed or that a packet  */
35 /* has been received and placed in a waiting buffer.                      */
36 /*                                                                        */
37 /* In lieu of a more conventional "on-the-fly" DMA mapping strategy in    */
38 /* which skbs are DMA mapped and immediately unmapped when the transmit   */
39 /* or receive has been completed, the VNIC driver is required to use      */
40 /* "long term mapping". This entails that large, continuous DMA mapped    */
41 /* buffers are allocated on driver initialization and these buffers are   */
42 /* then continuously reused to pass skbs to and from the VNIC server.     */
43 /*                                                                        */
44 /**************************************************************************/
45 
46 #include <linux/module.h>
47 #include <linux/moduleparam.h>
48 #include <linux/types.h>
49 #include <linux/errno.h>
50 #include <linux/completion.h>
51 #include <linux/ioport.h>
52 #include <linux/dma-mapping.h>
53 #include <linux/kernel.h>
54 #include <linux/netdevice.h>
55 #include <linux/etherdevice.h>
56 #include <linux/skbuff.h>
57 #include <linux/init.h>
58 #include <linux/delay.h>
59 #include <linux/mm.h>
60 #include <linux/ethtool.h>
61 #include <linux/proc_fs.h>
62 #include <linux/in.h>
63 #include <linux/ip.h>
64 #include <linux/ipv6.h>
65 #include <linux/irq.h>
66 #include <linux/kthread.h>
67 #include <linux/seq_file.h>
68 #include <linux/interrupt.h>
69 #include <net/net_namespace.h>
70 #include <asm/hvcall.h>
71 #include <linux/atomic.h>
72 #include <asm/vio.h>
73 #include <asm/iommu.h>
74 #include <linux/uaccess.h>
75 #include <asm/firmware.h>
76 #include <linux/workqueue.h>
77 #include <linux/if_vlan.h>
78 
79 #include "ibmvnic.h"
80 
81 static const char ibmvnic_driver_name[] = "ibmvnic";
82 static const char ibmvnic_driver_string[] = "IBM System i/p Virtual NIC Driver";
83 
84 MODULE_AUTHOR("Santiago Leon");
85 MODULE_DESCRIPTION("IBM System i/p Virtual NIC Driver");
86 MODULE_LICENSE("GPL");
87 MODULE_VERSION(IBMVNIC_DRIVER_VERSION);
88 
89 static int ibmvnic_version = IBMVNIC_INITIAL_VERSION;
90 static int ibmvnic_remove(struct vio_dev *);
91 static void release_sub_crqs(struct ibmvnic_adapter *);
92 static int ibmvnic_reset_crq(struct ibmvnic_adapter *);
93 static int ibmvnic_send_crq_init(struct ibmvnic_adapter *);
94 static int ibmvnic_reenable_crq_queue(struct ibmvnic_adapter *);
95 static int ibmvnic_send_crq(struct ibmvnic_adapter *, union ibmvnic_crq *);
96 static int send_subcrq(struct ibmvnic_adapter *adapter, u64 remote_handle,
97 		       union sub_crq *sub_crq);
98 static int send_subcrq_indirect(struct ibmvnic_adapter *, u64, u64, u64);
99 static irqreturn_t ibmvnic_interrupt_rx(int irq, void *instance);
100 static int enable_scrq_irq(struct ibmvnic_adapter *,
101 			   struct ibmvnic_sub_crq_queue *);
102 static int disable_scrq_irq(struct ibmvnic_adapter *,
103 			    struct ibmvnic_sub_crq_queue *);
104 static int pending_scrq(struct ibmvnic_adapter *,
105 			struct ibmvnic_sub_crq_queue *);
106 static union sub_crq *ibmvnic_next_scrq(struct ibmvnic_adapter *,
107 					struct ibmvnic_sub_crq_queue *);
108 static int ibmvnic_poll(struct napi_struct *napi, int data);
109 static void send_map_query(struct ibmvnic_adapter *adapter);
110 static void send_request_map(struct ibmvnic_adapter *, dma_addr_t, __be32, u8);
111 static void send_request_unmap(struct ibmvnic_adapter *, u8);
112 static void send_login(struct ibmvnic_adapter *adapter);
113 static void send_cap_queries(struct ibmvnic_adapter *adapter);
114 static int init_sub_crqs(struct ibmvnic_adapter *);
115 static int init_sub_crq_irqs(struct ibmvnic_adapter *adapter);
116 static int ibmvnic_init(struct ibmvnic_adapter *);
117 static void release_crq_queue(struct ibmvnic_adapter *);
118 
119 struct ibmvnic_stat {
120 	char name[ETH_GSTRING_LEN];
121 	int offset;
122 };
123 
124 #define IBMVNIC_STAT_OFF(stat) (offsetof(struct ibmvnic_adapter, stats) + \
125 			     offsetof(struct ibmvnic_statistics, stat))
126 #define IBMVNIC_GET_STAT(a, off) (*((u64 *)(((unsigned long)(a)) + off)))
127 
128 static const struct ibmvnic_stat ibmvnic_stats[] = {
129 	{"rx_packets", IBMVNIC_STAT_OFF(rx_packets)},
130 	{"rx_bytes", IBMVNIC_STAT_OFF(rx_bytes)},
131 	{"tx_packets", IBMVNIC_STAT_OFF(tx_packets)},
132 	{"tx_bytes", IBMVNIC_STAT_OFF(tx_bytes)},
133 	{"ucast_tx_packets", IBMVNIC_STAT_OFF(ucast_tx_packets)},
134 	{"ucast_rx_packets", IBMVNIC_STAT_OFF(ucast_rx_packets)},
135 	{"mcast_tx_packets", IBMVNIC_STAT_OFF(mcast_tx_packets)},
136 	{"mcast_rx_packets", IBMVNIC_STAT_OFF(mcast_rx_packets)},
137 	{"bcast_tx_packets", IBMVNIC_STAT_OFF(bcast_tx_packets)},
138 	{"bcast_rx_packets", IBMVNIC_STAT_OFF(bcast_rx_packets)},
139 	{"align_errors", IBMVNIC_STAT_OFF(align_errors)},
140 	{"fcs_errors", IBMVNIC_STAT_OFF(fcs_errors)},
141 	{"single_collision_frames", IBMVNIC_STAT_OFF(single_collision_frames)},
142 	{"multi_collision_frames", IBMVNIC_STAT_OFF(multi_collision_frames)},
143 	{"sqe_test_errors", IBMVNIC_STAT_OFF(sqe_test_errors)},
144 	{"deferred_tx", IBMVNIC_STAT_OFF(deferred_tx)},
145 	{"late_collisions", IBMVNIC_STAT_OFF(late_collisions)},
146 	{"excess_collisions", IBMVNIC_STAT_OFF(excess_collisions)},
147 	{"internal_mac_tx_errors", IBMVNIC_STAT_OFF(internal_mac_tx_errors)},
148 	{"carrier_sense", IBMVNIC_STAT_OFF(carrier_sense)},
149 	{"too_long_frames", IBMVNIC_STAT_OFF(too_long_frames)},
150 	{"internal_mac_rx_errors", IBMVNIC_STAT_OFF(internal_mac_rx_errors)},
151 };
152 
h_reg_sub_crq(unsigned long unit_address,unsigned long token,unsigned long length,unsigned long * number,unsigned long * irq)153 static long h_reg_sub_crq(unsigned long unit_address, unsigned long token,
154 			  unsigned long length, unsigned long *number,
155 			  unsigned long *irq)
156 {
157 	unsigned long retbuf[PLPAR_HCALL_BUFSIZE];
158 	long rc;
159 
160 	rc = plpar_hcall(H_REG_SUB_CRQ, retbuf, unit_address, token, length);
161 	*number = retbuf[0];
162 	*irq = retbuf[1];
163 
164 	return rc;
165 }
166 
alloc_long_term_buff(struct ibmvnic_adapter * adapter,struct ibmvnic_long_term_buff * ltb,int size)167 static int alloc_long_term_buff(struct ibmvnic_adapter *adapter,
168 				struct ibmvnic_long_term_buff *ltb, int size)
169 {
170 	struct device *dev = &adapter->vdev->dev;
171 
172 	ltb->size = size;
173 	ltb->buff = dma_alloc_coherent(dev, ltb->size, &ltb->addr,
174 				       GFP_KERNEL);
175 
176 	if (!ltb->buff) {
177 		dev_err(dev, "Couldn't alloc long term buffer\n");
178 		return -ENOMEM;
179 	}
180 	ltb->map_id = adapter->map_id;
181 	adapter->map_id++;
182 
183 	init_completion(&adapter->fw_done);
184 	send_request_map(adapter, ltb->addr,
185 			 ltb->size, ltb->map_id);
186 	wait_for_completion(&adapter->fw_done);
187 
188 	if (adapter->fw_done_rc) {
189 		dev_err(dev, "Couldn't map long term buffer,rc = %d\n",
190 			adapter->fw_done_rc);
191 		return -1;
192 	}
193 	return 0;
194 }
195 
free_long_term_buff(struct ibmvnic_adapter * adapter,struct ibmvnic_long_term_buff * ltb)196 static void free_long_term_buff(struct ibmvnic_adapter *adapter,
197 				struct ibmvnic_long_term_buff *ltb)
198 {
199 	struct device *dev = &adapter->vdev->dev;
200 
201 	if (!ltb->buff)
202 		return;
203 
204 	if (adapter->reset_reason != VNIC_RESET_FAILOVER &&
205 	    adapter->reset_reason != VNIC_RESET_MOBILITY)
206 		send_request_unmap(adapter, ltb->map_id);
207 	dma_free_coherent(dev, ltb->size, ltb->buff, ltb->addr);
208 }
209 
reset_long_term_buff(struct ibmvnic_adapter * adapter,struct ibmvnic_long_term_buff * ltb)210 static int reset_long_term_buff(struct ibmvnic_adapter *adapter,
211 				struct ibmvnic_long_term_buff *ltb)
212 {
213 	memset(ltb->buff, 0, ltb->size);
214 
215 	init_completion(&adapter->fw_done);
216 	send_request_map(adapter, ltb->addr, ltb->size, ltb->map_id);
217 	wait_for_completion(&adapter->fw_done);
218 
219 	if (adapter->fw_done_rc) {
220 		dev_info(&adapter->vdev->dev,
221 			 "Reset failed, attempting to free and reallocate buffer\n");
222 		free_long_term_buff(adapter, ltb);
223 		return alloc_long_term_buff(adapter, ltb, ltb->size);
224 	}
225 	return 0;
226 }
227 
deactivate_rx_pools(struct ibmvnic_adapter * adapter)228 static void deactivate_rx_pools(struct ibmvnic_adapter *adapter)
229 {
230 	int i;
231 
232 	for (i = 0; i < be32_to_cpu(adapter->login_rsp_buf->num_rxadd_subcrqs);
233 	     i++)
234 		adapter->rx_pool[i].active = 0;
235 }
236 
replenish_rx_pool(struct ibmvnic_adapter * adapter,struct ibmvnic_rx_pool * pool)237 static void replenish_rx_pool(struct ibmvnic_adapter *adapter,
238 			      struct ibmvnic_rx_pool *pool)
239 {
240 	int count = pool->size - atomic_read(&pool->available);
241 	struct device *dev = &adapter->vdev->dev;
242 	int buffers_added = 0;
243 	unsigned long lpar_rc;
244 	union sub_crq sub_crq;
245 	struct sk_buff *skb;
246 	unsigned int offset;
247 	dma_addr_t dma_addr;
248 	unsigned char *dst;
249 	u64 *handle_array;
250 	int shift = 0;
251 	int index;
252 	int i;
253 
254 	if (!pool->active)
255 		return;
256 
257 	handle_array = (u64 *)((u8 *)(adapter->login_rsp_buf) +
258 				      be32_to_cpu(adapter->login_rsp_buf->
259 				      off_rxadd_subcrqs));
260 
261 	for (i = 0; i < count; ++i) {
262 		skb = alloc_skb(pool->buff_size, GFP_ATOMIC);
263 		if (!skb) {
264 			dev_err(dev, "Couldn't replenish rx buff\n");
265 			adapter->replenish_no_mem++;
266 			break;
267 		}
268 
269 		index = pool->free_map[pool->next_free];
270 
271 		if (pool->rx_buff[index].skb)
272 			dev_err(dev, "Inconsistent free_map!\n");
273 
274 		/* Copy the skb to the long term mapped DMA buffer */
275 		offset = index * pool->buff_size;
276 		dst = pool->long_term_buff.buff + offset;
277 		memset(dst, 0, pool->buff_size);
278 		dma_addr = pool->long_term_buff.addr + offset;
279 		pool->rx_buff[index].data = dst;
280 
281 		pool->free_map[pool->next_free] = IBMVNIC_INVALID_MAP;
282 		pool->rx_buff[index].dma = dma_addr;
283 		pool->rx_buff[index].skb = skb;
284 		pool->rx_buff[index].pool_index = pool->index;
285 		pool->rx_buff[index].size = pool->buff_size;
286 
287 		memset(&sub_crq, 0, sizeof(sub_crq));
288 		sub_crq.rx_add.first = IBMVNIC_CRQ_CMD;
289 		sub_crq.rx_add.correlator =
290 		    cpu_to_be64((u64)&pool->rx_buff[index]);
291 		sub_crq.rx_add.ioba = cpu_to_be32(dma_addr);
292 		sub_crq.rx_add.map_id = pool->long_term_buff.map_id;
293 
294 		/* The length field of the sCRQ is defined to be 24 bits so the
295 		 * buffer size needs to be left shifted by a byte before it is
296 		 * converted to big endian to prevent the last byte from being
297 		 * truncated.
298 		 */
299 #ifdef __LITTLE_ENDIAN__
300 		shift = 8;
301 #endif
302 		sub_crq.rx_add.len = cpu_to_be32(pool->buff_size << shift);
303 
304 		lpar_rc = send_subcrq(adapter, handle_array[pool->index],
305 				      &sub_crq);
306 		if (lpar_rc != H_SUCCESS)
307 			goto failure;
308 
309 		buffers_added++;
310 		adapter->replenish_add_buff_success++;
311 		pool->next_free = (pool->next_free + 1) % pool->size;
312 	}
313 	atomic_add(buffers_added, &pool->available);
314 	return;
315 
316 failure:
317 	dev_info(dev, "replenish pools failure\n");
318 	pool->free_map[pool->next_free] = index;
319 	pool->rx_buff[index].skb = NULL;
320 	if (!dma_mapping_error(dev, dma_addr))
321 		dma_unmap_single(dev, dma_addr, pool->buff_size,
322 				 DMA_FROM_DEVICE);
323 
324 	dev_kfree_skb_any(skb);
325 	adapter->replenish_add_buff_failure++;
326 	atomic_add(buffers_added, &pool->available);
327 
328 	if (lpar_rc == H_CLOSED) {
329 		/* Disable buffer pool replenishment and report carrier off if
330 		 * queue is closed. Firmware guarantees that a signal will
331 		 * be sent to the driver, triggering a reset.
332 		 */
333 		deactivate_rx_pools(adapter);
334 		netif_carrier_off(adapter->netdev);
335 	}
336 }
337 
replenish_pools(struct ibmvnic_adapter * adapter)338 static void replenish_pools(struct ibmvnic_adapter *adapter)
339 {
340 	int i;
341 
342 	adapter->replenish_task_cycles++;
343 	for (i = 0; i < be32_to_cpu(adapter->login_rsp_buf->num_rxadd_subcrqs);
344 	     i++) {
345 		if (adapter->rx_pool[i].active)
346 			replenish_rx_pool(adapter, &adapter->rx_pool[i]);
347 	}
348 }
349 
release_stats_buffers(struct ibmvnic_adapter * adapter)350 static void release_stats_buffers(struct ibmvnic_adapter *adapter)
351 {
352 	kfree(adapter->tx_stats_buffers);
353 	kfree(adapter->rx_stats_buffers);
354 }
355 
init_stats_buffers(struct ibmvnic_adapter * adapter)356 static int init_stats_buffers(struct ibmvnic_adapter *adapter)
357 {
358 	adapter->tx_stats_buffers =
359 				kcalloc(adapter->req_tx_queues,
360 					sizeof(struct ibmvnic_tx_queue_stats),
361 					GFP_KERNEL);
362 	if (!adapter->tx_stats_buffers)
363 		return -ENOMEM;
364 
365 	adapter->rx_stats_buffers =
366 				kcalloc(adapter->req_rx_queues,
367 					sizeof(struct ibmvnic_rx_queue_stats),
368 					GFP_KERNEL);
369 	if (!adapter->rx_stats_buffers)
370 		return -ENOMEM;
371 
372 	return 0;
373 }
374 
release_stats_token(struct ibmvnic_adapter * adapter)375 static void release_stats_token(struct ibmvnic_adapter *adapter)
376 {
377 	struct device *dev = &adapter->vdev->dev;
378 
379 	if (!adapter->stats_token)
380 		return;
381 
382 	dma_unmap_single(dev, adapter->stats_token,
383 			 sizeof(struct ibmvnic_statistics),
384 			 DMA_FROM_DEVICE);
385 	adapter->stats_token = 0;
386 }
387 
init_stats_token(struct ibmvnic_adapter * adapter)388 static int init_stats_token(struct ibmvnic_adapter *adapter)
389 {
390 	struct device *dev = &adapter->vdev->dev;
391 	dma_addr_t stok;
392 
393 	stok = dma_map_single(dev, &adapter->stats,
394 			      sizeof(struct ibmvnic_statistics),
395 			      DMA_FROM_DEVICE);
396 	if (dma_mapping_error(dev, stok)) {
397 		dev_err(dev, "Couldn't map stats buffer\n");
398 		return -1;
399 	}
400 
401 	adapter->stats_token = stok;
402 	netdev_dbg(adapter->netdev, "Stats token initialized (%llx)\n", stok);
403 	return 0;
404 }
405 
reset_rx_pools(struct ibmvnic_adapter * adapter)406 static int reset_rx_pools(struct ibmvnic_adapter *adapter)
407 {
408 	struct ibmvnic_rx_pool *rx_pool;
409 	int rx_scrqs;
410 	int i, j, rc;
411 
412 	rx_scrqs = be32_to_cpu(adapter->login_rsp_buf->num_rxadd_subcrqs);
413 	for (i = 0; i < rx_scrqs; i++) {
414 		rx_pool = &adapter->rx_pool[i];
415 
416 		netdev_dbg(adapter->netdev, "Re-setting rx_pool[%d]\n", i);
417 
418 		rc = reset_long_term_buff(adapter, &rx_pool->long_term_buff);
419 		if (rc)
420 			return rc;
421 
422 		for (j = 0; j < rx_pool->size; j++)
423 			rx_pool->free_map[j] = j;
424 
425 		memset(rx_pool->rx_buff, 0,
426 		       rx_pool->size * sizeof(struct ibmvnic_rx_buff));
427 
428 		atomic_set(&rx_pool->available, 0);
429 		rx_pool->next_alloc = 0;
430 		rx_pool->next_free = 0;
431 		rx_pool->active = 1;
432 	}
433 
434 	return 0;
435 }
436 
release_rx_pools(struct ibmvnic_adapter * adapter)437 static void release_rx_pools(struct ibmvnic_adapter *adapter)
438 {
439 	struct ibmvnic_rx_pool *rx_pool;
440 	int rx_scrqs;
441 	int i, j;
442 
443 	if (!adapter->rx_pool)
444 		return;
445 
446 	rx_scrqs = be32_to_cpu(adapter->login_rsp_buf->num_rxadd_subcrqs);
447 	for (i = 0; i < rx_scrqs; i++) {
448 		rx_pool = &adapter->rx_pool[i];
449 
450 		netdev_dbg(adapter->netdev, "Releasing rx_pool[%d]\n", i);
451 
452 		kfree(rx_pool->free_map);
453 		free_long_term_buff(adapter, &rx_pool->long_term_buff);
454 
455 		if (!rx_pool->rx_buff)
456 			continue;
457 
458 		for (j = 0; j < rx_pool->size; j++) {
459 			if (rx_pool->rx_buff[j].skb) {
460 				dev_kfree_skb_any(rx_pool->rx_buff[j].skb);
461 				rx_pool->rx_buff[j].skb = NULL;
462 			}
463 		}
464 
465 		kfree(rx_pool->rx_buff);
466 	}
467 
468 	kfree(adapter->rx_pool);
469 	adapter->rx_pool = NULL;
470 }
471 
init_rx_pools(struct net_device * netdev)472 static int init_rx_pools(struct net_device *netdev)
473 {
474 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
475 	struct device *dev = &adapter->vdev->dev;
476 	struct ibmvnic_rx_pool *rx_pool;
477 	int rxadd_subcrqs;
478 	u64 *size_array;
479 	int i, j;
480 
481 	rxadd_subcrqs =
482 		be32_to_cpu(adapter->login_rsp_buf->num_rxadd_subcrqs);
483 	size_array = (u64 *)((u8 *)(adapter->login_rsp_buf) +
484 		be32_to_cpu(adapter->login_rsp_buf->off_rxadd_buff_size));
485 
486 	adapter->rx_pool = kcalloc(rxadd_subcrqs,
487 				   sizeof(struct ibmvnic_rx_pool),
488 				   GFP_KERNEL);
489 	if (!adapter->rx_pool) {
490 		dev_err(dev, "Failed to allocate rx pools\n");
491 		return -1;
492 	}
493 
494 	for (i = 0; i < rxadd_subcrqs; i++) {
495 		rx_pool = &adapter->rx_pool[i];
496 
497 		netdev_dbg(adapter->netdev,
498 			   "Initializing rx_pool[%d], %lld buffs, %lld bytes each\n",
499 			   i, adapter->req_rx_add_entries_per_subcrq,
500 			   be64_to_cpu(size_array[i]));
501 
502 		rx_pool->size = adapter->req_rx_add_entries_per_subcrq;
503 		rx_pool->index = i;
504 		rx_pool->buff_size = be64_to_cpu(size_array[i]);
505 		rx_pool->active = 1;
506 
507 		rx_pool->free_map = kcalloc(rx_pool->size, sizeof(int),
508 					    GFP_KERNEL);
509 		if (!rx_pool->free_map) {
510 			release_rx_pools(adapter);
511 			return -1;
512 		}
513 
514 		rx_pool->rx_buff = kcalloc(rx_pool->size,
515 					   sizeof(struct ibmvnic_rx_buff),
516 					   GFP_KERNEL);
517 		if (!rx_pool->rx_buff) {
518 			dev_err(dev, "Couldn't alloc rx buffers\n");
519 			release_rx_pools(adapter);
520 			return -1;
521 		}
522 
523 		if (alloc_long_term_buff(adapter, &rx_pool->long_term_buff,
524 					 rx_pool->size * rx_pool->buff_size)) {
525 			release_rx_pools(adapter);
526 			return -1;
527 		}
528 
529 		for (j = 0; j < rx_pool->size; ++j)
530 			rx_pool->free_map[j] = j;
531 
532 		atomic_set(&rx_pool->available, 0);
533 		rx_pool->next_alloc = 0;
534 		rx_pool->next_free = 0;
535 	}
536 
537 	return 0;
538 }
539 
reset_tx_pools(struct ibmvnic_adapter * adapter)540 static int reset_tx_pools(struct ibmvnic_adapter *adapter)
541 {
542 	struct ibmvnic_tx_pool *tx_pool;
543 	int tx_scrqs;
544 	int i, j, rc;
545 
546 	tx_scrqs = be32_to_cpu(adapter->login_rsp_buf->num_txsubm_subcrqs);
547 	for (i = 0; i < tx_scrqs; i++) {
548 		netdev_dbg(adapter->netdev, "Re-setting tx_pool[%d]\n", i);
549 
550 		tx_pool = &adapter->tx_pool[i];
551 
552 		rc = reset_long_term_buff(adapter, &tx_pool->long_term_buff);
553 		if (rc)
554 			return rc;
555 
556 		memset(tx_pool->tx_buff, 0,
557 		       adapter->req_tx_entries_per_subcrq *
558 		       sizeof(struct ibmvnic_tx_buff));
559 
560 		for (j = 0; j < adapter->req_tx_entries_per_subcrq; j++)
561 			tx_pool->free_map[j] = j;
562 
563 		tx_pool->consumer_index = 0;
564 		tx_pool->producer_index = 0;
565 	}
566 
567 	return 0;
568 }
569 
release_tx_pools(struct ibmvnic_adapter * adapter)570 static void release_tx_pools(struct ibmvnic_adapter *adapter)
571 {
572 	struct ibmvnic_tx_pool *tx_pool;
573 	int i, tx_scrqs;
574 
575 	if (!adapter->tx_pool)
576 		return;
577 
578 	tx_scrqs = be32_to_cpu(adapter->login_rsp_buf->num_txsubm_subcrqs);
579 	for (i = 0; i < tx_scrqs; i++) {
580 		netdev_dbg(adapter->netdev, "Releasing tx_pool[%d]\n", i);
581 		tx_pool = &adapter->tx_pool[i];
582 		kfree(tx_pool->tx_buff);
583 		free_long_term_buff(adapter, &tx_pool->long_term_buff);
584 		kfree(tx_pool->free_map);
585 	}
586 
587 	kfree(adapter->tx_pool);
588 	adapter->tx_pool = NULL;
589 }
590 
init_tx_pools(struct net_device * netdev)591 static int init_tx_pools(struct net_device *netdev)
592 {
593 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
594 	struct device *dev = &adapter->vdev->dev;
595 	struct ibmvnic_tx_pool *tx_pool;
596 	int tx_subcrqs;
597 	int i, j;
598 
599 	tx_subcrqs = be32_to_cpu(adapter->login_rsp_buf->num_txsubm_subcrqs);
600 	adapter->tx_pool = kcalloc(tx_subcrqs,
601 				   sizeof(struct ibmvnic_tx_pool), GFP_KERNEL);
602 	if (!adapter->tx_pool)
603 		return -1;
604 
605 	for (i = 0; i < tx_subcrqs; i++) {
606 		tx_pool = &adapter->tx_pool[i];
607 
608 		netdev_dbg(adapter->netdev,
609 			   "Initializing tx_pool[%d], %lld buffs\n",
610 			   i, adapter->req_tx_entries_per_subcrq);
611 
612 		tx_pool->tx_buff = kcalloc(adapter->req_tx_entries_per_subcrq,
613 					   sizeof(struct ibmvnic_tx_buff),
614 					   GFP_KERNEL);
615 		if (!tx_pool->tx_buff) {
616 			dev_err(dev, "tx pool buffer allocation failed\n");
617 			release_tx_pools(adapter);
618 			return -1;
619 		}
620 
621 		if (alloc_long_term_buff(adapter, &tx_pool->long_term_buff,
622 					 adapter->req_tx_entries_per_subcrq *
623 					 adapter->req_mtu)) {
624 			release_tx_pools(adapter);
625 			return -1;
626 		}
627 
628 		tx_pool->free_map = kcalloc(adapter->req_tx_entries_per_subcrq,
629 					    sizeof(int), GFP_KERNEL);
630 		if (!tx_pool->free_map) {
631 			release_tx_pools(adapter);
632 			return -1;
633 		}
634 
635 		for (j = 0; j < adapter->req_tx_entries_per_subcrq; j++)
636 			tx_pool->free_map[j] = j;
637 
638 		tx_pool->consumer_index = 0;
639 		tx_pool->producer_index = 0;
640 	}
641 
642 	return 0;
643 }
644 
release_error_buffers(struct ibmvnic_adapter * adapter)645 static void release_error_buffers(struct ibmvnic_adapter *adapter)
646 {
647 	struct device *dev = &adapter->vdev->dev;
648 	struct ibmvnic_error_buff *error_buff, *tmp;
649 	unsigned long flags;
650 
651 	spin_lock_irqsave(&adapter->error_list_lock, flags);
652 	list_for_each_entry_safe(error_buff, tmp, &adapter->errors, list) {
653 		list_del(&error_buff->list);
654 		dma_unmap_single(dev, error_buff->dma, error_buff->len,
655 				 DMA_FROM_DEVICE);
656 		kfree(error_buff->buff);
657 		kfree(error_buff);
658 	}
659 	spin_unlock_irqrestore(&adapter->error_list_lock, flags);
660 }
661 
ibmvnic_napi_enable(struct ibmvnic_adapter * adapter)662 static void ibmvnic_napi_enable(struct ibmvnic_adapter *adapter)
663 {
664 	int i;
665 
666 	if (adapter->napi_enabled)
667 		return;
668 
669 	for (i = 0; i < adapter->req_rx_queues; i++)
670 		napi_enable(&adapter->napi[i]);
671 
672 	adapter->napi_enabled = true;
673 }
674 
ibmvnic_napi_disable(struct ibmvnic_adapter * adapter)675 static void ibmvnic_napi_disable(struct ibmvnic_adapter *adapter)
676 {
677 	int i;
678 
679 	if (!adapter->napi_enabled)
680 		return;
681 
682 	for (i = 0; i < adapter->req_rx_queues; i++) {
683 		netdev_dbg(adapter->netdev, "Disabling napi[%d]\n", i);
684 		napi_disable(&adapter->napi[i]);
685 	}
686 
687 	adapter->napi_enabled = false;
688 }
689 
ibmvnic_login(struct net_device * netdev)690 static int ibmvnic_login(struct net_device *netdev)
691 {
692 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
693 	unsigned long timeout = msecs_to_jiffies(30000);
694 	struct device *dev = &adapter->vdev->dev;
695 	int rc;
696 
697 	do {
698 		if (adapter->renegotiate) {
699 			adapter->renegotiate = false;
700 			release_sub_crqs(adapter);
701 
702 			reinit_completion(&adapter->init_done);
703 			send_cap_queries(adapter);
704 			if (!wait_for_completion_timeout(&adapter->init_done,
705 							 timeout)) {
706 				dev_err(dev, "Capabilities query timeout\n");
707 				return -1;
708 			}
709 			rc = init_sub_crqs(adapter);
710 			if (rc) {
711 				dev_err(dev,
712 					"Initialization of SCRQ's failed\n");
713 				return -1;
714 			}
715 			rc = init_sub_crq_irqs(adapter);
716 			if (rc) {
717 				dev_err(dev,
718 					"Initialization of SCRQ's irqs failed\n");
719 				return -1;
720 			}
721 		}
722 
723 		reinit_completion(&adapter->init_done);
724 		send_login(adapter);
725 		if (!wait_for_completion_timeout(&adapter->init_done,
726 						 timeout)) {
727 			dev_err(dev, "Login timeout\n");
728 			return -1;
729 		}
730 	} while (adapter->renegotiate);
731 
732 	return 0;
733 }
734 
release_resources(struct ibmvnic_adapter * adapter)735 static void release_resources(struct ibmvnic_adapter *adapter)
736 {
737 	int i;
738 
739 	release_tx_pools(adapter);
740 	release_rx_pools(adapter);
741 
742 	release_stats_token(adapter);
743 	release_stats_buffers(adapter);
744 	release_error_buffers(adapter);
745 
746 	if (adapter->napi) {
747 		for (i = 0; i < adapter->req_rx_queues; i++) {
748 			if (&adapter->napi[i]) {
749 				netdev_dbg(adapter->netdev,
750 					   "Releasing napi[%d]\n", i);
751 				netif_napi_del(&adapter->napi[i]);
752 			}
753 		}
754 	}
755 }
756 
set_link_state(struct ibmvnic_adapter * adapter,u8 link_state)757 static int set_link_state(struct ibmvnic_adapter *adapter, u8 link_state)
758 {
759 	struct net_device *netdev = adapter->netdev;
760 	unsigned long timeout = msecs_to_jiffies(30000);
761 	union ibmvnic_crq crq;
762 	bool resend;
763 	int rc;
764 
765 	netdev_dbg(netdev, "setting link state %d\n", link_state);
766 
767 	memset(&crq, 0, sizeof(crq));
768 	crq.logical_link_state.first = IBMVNIC_CRQ_CMD;
769 	crq.logical_link_state.cmd = LOGICAL_LINK_STATE;
770 	crq.logical_link_state.link_state = link_state;
771 
772 	do {
773 		resend = false;
774 
775 		reinit_completion(&adapter->init_done);
776 		rc = ibmvnic_send_crq(adapter, &crq);
777 		if (rc) {
778 			netdev_err(netdev, "Failed to set link state\n");
779 			return rc;
780 		}
781 
782 		if (!wait_for_completion_timeout(&adapter->init_done,
783 						 timeout)) {
784 			netdev_err(netdev, "timeout setting link state\n");
785 			return -1;
786 		}
787 
788 		if (adapter->init_done_rc == 1) {
789 			/* Partuial success, delay and re-send */
790 			mdelay(1000);
791 			resend = true;
792 		}
793 	} while (resend);
794 
795 	return 0;
796 }
797 
set_real_num_queues(struct net_device * netdev)798 static int set_real_num_queues(struct net_device *netdev)
799 {
800 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
801 	int rc;
802 
803 	netdev_dbg(netdev, "Setting real tx/rx queues (%llx/%llx)\n",
804 		   adapter->req_tx_queues, adapter->req_rx_queues);
805 
806 	rc = netif_set_real_num_tx_queues(netdev, adapter->req_tx_queues);
807 	if (rc) {
808 		netdev_err(netdev, "failed to set the number of tx queues\n");
809 		return rc;
810 	}
811 
812 	rc = netif_set_real_num_rx_queues(netdev, adapter->req_rx_queues);
813 	if (rc)
814 		netdev_err(netdev, "failed to set the number of rx queues\n");
815 
816 	return rc;
817 }
818 
init_resources(struct ibmvnic_adapter * adapter)819 static int init_resources(struct ibmvnic_adapter *adapter)
820 {
821 	struct net_device *netdev = adapter->netdev;
822 	int i, rc;
823 
824 	rc = set_real_num_queues(netdev);
825 	if (rc)
826 		return rc;
827 
828 	rc = init_stats_buffers(adapter);
829 	if (rc)
830 		return rc;
831 
832 	rc = init_stats_token(adapter);
833 	if (rc)
834 		return rc;
835 
836 	adapter->map_id = 1;
837 	adapter->napi = kcalloc(adapter->req_rx_queues,
838 				sizeof(struct napi_struct), GFP_KERNEL);
839 	if (!adapter->napi)
840 		return -ENOMEM;
841 
842 	for (i = 0; i < adapter->req_rx_queues; i++) {
843 		netdev_dbg(netdev, "Adding napi[%d]\n", i);
844 		netif_napi_add(netdev, &adapter->napi[i], ibmvnic_poll,
845 			       NAPI_POLL_WEIGHT);
846 	}
847 
848 	send_map_query(adapter);
849 
850 	rc = init_rx_pools(netdev);
851 	if (rc)
852 		return rc;
853 
854 	rc = init_tx_pools(netdev);
855 	return rc;
856 }
857 
__ibmvnic_open(struct net_device * netdev)858 static int __ibmvnic_open(struct net_device *netdev)
859 {
860 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
861 	enum vnic_state prev_state = adapter->state;
862 	int i, rc;
863 
864 	adapter->state = VNIC_OPENING;
865 	replenish_pools(adapter);
866 	ibmvnic_napi_enable(adapter);
867 
868 	/* We're ready to receive frames, enable the sub-crq interrupts and
869 	 * set the logical link state to up
870 	 */
871 	for (i = 0; i < adapter->req_rx_queues; i++) {
872 		netdev_dbg(netdev, "Enabling rx_scrq[%d] irq\n", i);
873 		if (prev_state == VNIC_CLOSED)
874 			enable_irq(adapter->rx_scrq[i]->irq);
875 		else
876 			enable_scrq_irq(adapter, adapter->rx_scrq[i]);
877 	}
878 
879 	for (i = 0; i < adapter->req_tx_queues; i++) {
880 		netdev_dbg(netdev, "Enabling tx_scrq[%d] irq\n", i);
881 		if (prev_state == VNIC_CLOSED)
882 			enable_irq(adapter->tx_scrq[i]->irq);
883 		else
884 			enable_scrq_irq(adapter, adapter->tx_scrq[i]);
885 	}
886 
887 	rc = set_link_state(adapter, IBMVNIC_LOGICAL_LNK_UP);
888 	if (rc) {
889 		for (i = 0; i < adapter->req_rx_queues; i++)
890 			napi_disable(&adapter->napi[i]);
891 		release_resources(adapter);
892 		return rc;
893 	}
894 
895 	netif_tx_start_all_queues(netdev);
896 
897 	if (prev_state == VNIC_CLOSED) {
898 		for (i = 0; i < adapter->req_rx_queues; i++)
899 			napi_schedule(&adapter->napi[i]);
900 	}
901 
902 	adapter->state = VNIC_OPEN;
903 	return rc;
904 }
905 
ibmvnic_open(struct net_device * netdev)906 static int ibmvnic_open(struct net_device *netdev)
907 {
908 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
909 	int rc;
910 
911 	mutex_lock(&adapter->reset_lock);
912 
913 	if (adapter->state != VNIC_CLOSED) {
914 		rc = ibmvnic_login(netdev);
915 		if (rc) {
916 			mutex_unlock(&adapter->reset_lock);
917 			return rc;
918 		}
919 
920 		rc = init_resources(adapter);
921 		if (rc) {
922 			netdev_err(netdev, "failed to initialize resources\n");
923 			release_resources(adapter);
924 			mutex_unlock(&adapter->reset_lock);
925 			return rc;
926 		}
927 	}
928 
929 	rc = __ibmvnic_open(netdev);
930 	netif_carrier_on(netdev);
931 	mutex_unlock(&adapter->reset_lock);
932 
933 	return rc;
934 }
935 
clean_rx_pools(struct ibmvnic_adapter * adapter)936 static void clean_rx_pools(struct ibmvnic_adapter *adapter)
937 {
938 	struct ibmvnic_rx_pool *rx_pool;
939 	u64 rx_entries;
940 	int rx_scrqs;
941 	int i, j;
942 
943 	if (!adapter->rx_pool)
944 		return;
945 
946 	rx_scrqs = be32_to_cpu(adapter->login_rsp_buf->num_rxadd_subcrqs);
947 	rx_entries = adapter->req_rx_add_entries_per_subcrq;
948 
949 	/* Free any remaining skbs in the rx buffer pools */
950 	for (i = 0; i < rx_scrqs; i++) {
951 		rx_pool = &adapter->rx_pool[i];
952 		if (!rx_pool)
953 			continue;
954 
955 		netdev_dbg(adapter->netdev, "Cleaning rx_pool[%d]\n", i);
956 		for (j = 0; j < rx_entries; j++) {
957 			if (rx_pool->rx_buff[j].skb) {
958 				dev_kfree_skb_any(rx_pool->rx_buff[j].skb);
959 				rx_pool->rx_buff[j].skb = NULL;
960 			}
961 		}
962 	}
963 }
964 
clean_tx_pools(struct ibmvnic_adapter * adapter)965 static void clean_tx_pools(struct ibmvnic_adapter *adapter)
966 {
967 	struct ibmvnic_tx_pool *tx_pool;
968 	u64 tx_entries;
969 	int tx_scrqs;
970 	int i, j;
971 
972 	if (!adapter->tx_pool)
973 		return;
974 
975 	tx_scrqs = be32_to_cpu(adapter->login_rsp_buf->num_txsubm_subcrqs);
976 	tx_entries = adapter->req_tx_entries_per_subcrq;
977 
978 	/* Free any remaining skbs in the tx buffer pools */
979 	for (i = 0; i < tx_scrqs; i++) {
980 		tx_pool = &adapter->tx_pool[i];
981 		if (!tx_pool)
982 			continue;
983 
984 		netdev_dbg(adapter->netdev, "Cleaning tx_pool[%d]\n", i);
985 		for (j = 0; j < tx_entries; j++) {
986 			if (tx_pool->tx_buff[j].skb) {
987 				dev_kfree_skb_any(tx_pool->tx_buff[j].skb);
988 				tx_pool->tx_buff[j].skb = NULL;
989 			}
990 		}
991 	}
992 }
993 
__ibmvnic_close(struct net_device * netdev)994 static int __ibmvnic_close(struct net_device *netdev)
995 {
996 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
997 	int rc = 0;
998 	int i;
999 
1000 	adapter->state = VNIC_CLOSING;
1001 
1002 	/* ensure that transmissions are stopped if called by do_reset */
1003 	if (adapter->resetting)
1004 		netif_tx_disable(netdev);
1005 	else
1006 		netif_tx_stop_all_queues(netdev);
1007 
1008 	ibmvnic_napi_disable(adapter);
1009 
1010 	if (adapter->tx_scrq) {
1011 		for (i = 0; i < adapter->req_tx_queues; i++)
1012 			if (adapter->tx_scrq[i]->irq) {
1013 				netdev_dbg(adapter->netdev,
1014 					   "Disabling tx_scrq[%d] irq\n", i);
1015 				disable_irq(adapter->tx_scrq[i]->irq);
1016 			}
1017 	}
1018 
1019 	rc = set_link_state(adapter, IBMVNIC_LOGICAL_LNK_DN);
1020 	if (rc)
1021 		return rc;
1022 
1023 	if (adapter->rx_scrq) {
1024 		for (i = 0; i < adapter->req_rx_queues; i++) {
1025 			int retries = 10;
1026 
1027 			while (pending_scrq(adapter, adapter->rx_scrq[i])) {
1028 				retries--;
1029 				mdelay(100);
1030 
1031 				if (retries == 0)
1032 					break;
1033 			}
1034 
1035 			if (adapter->rx_scrq[i]->irq) {
1036 				netdev_dbg(adapter->netdev,
1037 					   "Disabling rx_scrq[%d] irq\n", i);
1038 				disable_irq(adapter->rx_scrq[i]->irq);
1039 			}
1040 		}
1041 	}
1042 	clean_rx_pools(adapter);
1043 	clean_tx_pools(adapter);
1044 	adapter->state = VNIC_CLOSED;
1045 	return rc;
1046 }
1047 
ibmvnic_close(struct net_device * netdev)1048 static int ibmvnic_close(struct net_device *netdev)
1049 {
1050 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
1051 	int rc;
1052 
1053 	mutex_lock(&adapter->reset_lock);
1054 	rc = __ibmvnic_close(netdev);
1055 	mutex_unlock(&adapter->reset_lock);
1056 
1057 	return rc;
1058 }
1059 
1060 /**
1061  * build_hdr_data - creates L2/L3/L4 header data buffer
1062  * @hdr_field - bitfield determining needed headers
1063  * @skb - socket buffer
1064  * @hdr_len - array of header lengths
1065  * @tot_len - total length of data
1066  *
1067  * Reads hdr_field to determine which headers are needed by firmware.
1068  * Builds a buffer containing these headers.  Saves individual header
1069  * lengths and total buffer length to be used to build descriptors.
1070  */
build_hdr_data(u8 hdr_field,struct sk_buff * skb,int * hdr_len,u8 * hdr_data)1071 static int build_hdr_data(u8 hdr_field, struct sk_buff *skb,
1072 			  int *hdr_len, u8 *hdr_data)
1073 {
1074 	int len = 0;
1075 	u8 *hdr;
1076 
1077 	hdr_len[0] = sizeof(struct ethhdr);
1078 
1079 	if (skb->protocol == htons(ETH_P_IP)) {
1080 		hdr_len[1] = ip_hdr(skb)->ihl * 4;
1081 		if (ip_hdr(skb)->protocol == IPPROTO_TCP)
1082 			hdr_len[2] = tcp_hdrlen(skb);
1083 		else if (ip_hdr(skb)->protocol == IPPROTO_UDP)
1084 			hdr_len[2] = sizeof(struct udphdr);
1085 	} else if (skb->protocol == htons(ETH_P_IPV6)) {
1086 		hdr_len[1] = sizeof(struct ipv6hdr);
1087 		if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
1088 			hdr_len[2] = tcp_hdrlen(skb);
1089 		else if (ipv6_hdr(skb)->nexthdr == IPPROTO_UDP)
1090 			hdr_len[2] = sizeof(struct udphdr);
1091 	}
1092 
1093 	memset(hdr_data, 0, 120);
1094 	if ((hdr_field >> 6) & 1) {
1095 		hdr = skb_mac_header(skb);
1096 		memcpy(hdr_data, hdr, hdr_len[0]);
1097 		len += hdr_len[0];
1098 	}
1099 
1100 	if ((hdr_field >> 5) & 1) {
1101 		hdr = skb_network_header(skb);
1102 		memcpy(hdr_data + len, hdr, hdr_len[1]);
1103 		len += hdr_len[1];
1104 	}
1105 
1106 	if ((hdr_field >> 4) & 1) {
1107 		hdr = skb_transport_header(skb);
1108 		memcpy(hdr_data + len, hdr, hdr_len[2]);
1109 		len += hdr_len[2];
1110 	}
1111 	return len;
1112 }
1113 
1114 /**
1115  * create_hdr_descs - create header and header extension descriptors
1116  * @hdr_field - bitfield determining needed headers
1117  * @data - buffer containing header data
1118  * @len - length of data buffer
1119  * @hdr_len - array of individual header lengths
1120  * @scrq_arr - descriptor array
1121  *
1122  * Creates header and, if needed, header extension descriptors and
1123  * places them in a descriptor array, scrq_arr
1124  */
1125 
create_hdr_descs(u8 hdr_field,u8 * hdr_data,int len,int * hdr_len,union sub_crq * scrq_arr)1126 static int create_hdr_descs(u8 hdr_field, u8 *hdr_data, int len, int *hdr_len,
1127 			    union sub_crq *scrq_arr)
1128 {
1129 	union sub_crq hdr_desc;
1130 	int tmp_len = len;
1131 	int num_descs = 0;
1132 	u8 *data, *cur;
1133 	int tmp;
1134 
1135 	while (tmp_len > 0) {
1136 		cur = hdr_data + len - tmp_len;
1137 
1138 		memset(&hdr_desc, 0, sizeof(hdr_desc));
1139 		if (cur != hdr_data) {
1140 			data = hdr_desc.hdr_ext.data;
1141 			tmp = tmp_len > 29 ? 29 : tmp_len;
1142 			hdr_desc.hdr_ext.first = IBMVNIC_CRQ_CMD;
1143 			hdr_desc.hdr_ext.type = IBMVNIC_HDR_EXT_DESC;
1144 			hdr_desc.hdr_ext.len = tmp;
1145 		} else {
1146 			data = hdr_desc.hdr.data;
1147 			tmp = tmp_len > 24 ? 24 : tmp_len;
1148 			hdr_desc.hdr.first = IBMVNIC_CRQ_CMD;
1149 			hdr_desc.hdr.type = IBMVNIC_HDR_DESC;
1150 			hdr_desc.hdr.len = tmp;
1151 			hdr_desc.hdr.l2_len = (u8)hdr_len[0];
1152 			hdr_desc.hdr.l3_len = cpu_to_be16((u16)hdr_len[1]);
1153 			hdr_desc.hdr.l4_len = (u8)hdr_len[2];
1154 			hdr_desc.hdr.flag = hdr_field << 1;
1155 		}
1156 		memcpy(data, cur, tmp);
1157 		tmp_len -= tmp;
1158 		*scrq_arr = hdr_desc;
1159 		scrq_arr++;
1160 		num_descs++;
1161 	}
1162 
1163 	return num_descs;
1164 }
1165 
1166 /**
1167  * build_hdr_descs_arr - build a header descriptor array
1168  * @skb - socket buffer
1169  * @num_entries - number of descriptors to be sent
1170  * @subcrq - first TX descriptor
1171  * @hdr_field - bit field determining which headers will be sent
1172  *
1173  * This function will build a TX descriptor array with applicable
1174  * L2/L3/L4 packet header descriptors to be sent by send_subcrq_indirect.
1175  */
1176 
build_hdr_descs_arr(struct ibmvnic_tx_buff * txbuff,int * num_entries,u8 hdr_field)1177 static void build_hdr_descs_arr(struct ibmvnic_tx_buff *txbuff,
1178 				int *num_entries, u8 hdr_field)
1179 {
1180 	int hdr_len[3] = {0, 0, 0};
1181 	int tot_len;
1182 	u8 *hdr_data = txbuff->hdr_data;
1183 
1184 	tot_len = build_hdr_data(hdr_field, txbuff->skb, hdr_len,
1185 				 txbuff->hdr_data);
1186 	*num_entries += create_hdr_descs(hdr_field, hdr_data, tot_len, hdr_len,
1187 			 txbuff->indir_arr + 1);
1188 }
1189 
ibmvnic_xmit(struct sk_buff * skb,struct net_device * netdev)1190 static int ibmvnic_xmit(struct sk_buff *skb, struct net_device *netdev)
1191 {
1192 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
1193 	int queue_num = skb_get_queue_mapping(skb);
1194 	u8 *hdrs = (u8 *)&adapter->tx_rx_desc_req;
1195 	struct device *dev = &adapter->vdev->dev;
1196 	struct ibmvnic_tx_buff *tx_buff = NULL;
1197 	struct ibmvnic_sub_crq_queue *tx_scrq;
1198 	struct ibmvnic_tx_pool *tx_pool;
1199 	unsigned int tx_send_failed = 0;
1200 	unsigned int tx_map_failed = 0;
1201 	unsigned int tx_dropped = 0;
1202 	unsigned int tx_packets = 0;
1203 	unsigned int tx_bytes = 0;
1204 	dma_addr_t data_dma_addr;
1205 	struct netdev_queue *txq;
1206 	unsigned long lpar_rc;
1207 	union sub_crq tx_crq;
1208 	unsigned int offset;
1209 	int num_entries = 1;
1210 	unsigned char *dst;
1211 	u64 *handle_array;
1212 	int index = 0;
1213 	int ret = 0;
1214 
1215 	if (adapter->resetting) {
1216 		if (!netif_subqueue_stopped(netdev, skb))
1217 			netif_stop_subqueue(netdev, queue_num);
1218 		dev_kfree_skb_any(skb);
1219 
1220 		tx_send_failed++;
1221 		tx_dropped++;
1222 		ret = NETDEV_TX_OK;
1223 		goto out;
1224 	}
1225 
1226 	tx_pool = &adapter->tx_pool[queue_num];
1227 	tx_scrq = adapter->tx_scrq[queue_num];
1228 	txq = netdev_get_tx_queue(netdev, skb_get_queue_mapping(skb));
1229 	handle_array = (u64 *)((u8 *)(adapter->login_rsp_buf) +
1230 		be32_to_cpu(adapter->login_rsp_buf->off_txsubm_subcrqs));
1231 
1232 	index = tx_pool->free_map[tx_pool->consumer_index];
1233 	offset = index * adapter->req_mtu;
1234 	dst = tx_pool->long_term_buff.buff + offset;
1235 	memset(dst, 0, adapter->req_mtu);
1236 	skb_copy_from_linear_data(skb, dst, skb->len);
1237 	data_dma_addr = tx_pool->long_term_buff.addr + offset;
1238 
1239 	tx_pool->consumer_index =
1240 	    (tx_pool->consumer_index + 1) %
1241 		adapter->req_tx_entries_per_subcrq;
1242 
1243 	tx_buff = &tx_pool->tx_buff[index];
1244 	tx_buff->skb = skb;
1245 	tx_buff->data_dma[0] = data_dma_addr;
1246 	tx_buff->data_len[0] = skb->len;
1247 	tx_buff->index = index;
1248 	tx_buff->pool_index = queue_num;
1249 	tx_buff->last_frag = true;
1250 
1251 	memset(&tx_crq, 0, sizeof(tx_crq));
1252 	tx_crq.v1.first = IBMVNIC_CRQ_CMD;
1253 	tx_crq.v1.type = IBMVNIC_TX_DESC;
1254 	tx_crq.v1.n_crq_elem = 1;
1255 	tx_crq.v1.n_sge = 1;
1256 	tx_crq.v1.flags1 = IBMVNIC_TX_COMP_NEEDED;
1257 	tx_crq.v1.correlator = cpu_to_be32(index);
1258 	tx_crq.v1.dma_reg = cpu_to_be16(tx_pool->long_term_buff.map_id);
1259 	tx_crq.v1.sge_len = cpu_to_be32(skb->len);
1260 	tx_crq.v1.ioba = cpu_to_be64(data_dma_addr);
1261 
1262 	if (adapter->vlan_header_insertion && skb_vlan_tag_present(skb)) {
1263 		tx_crq.v1.flags2 |= IBMVNIC_TX_VLAN_INSERT;
1264 		tx_crq.v1.vlan_id = cpu_to_be16(skb->vlan_tci);
1265 	}
1266 
1267 	if (skb->protocol == htons(ETH_P_IP)) {
1268 		if (ip_hdr(skb)->version == 4)
1269 			tx_crq.v1.flags1 |= IBMVNIC_TX_PROT_IPV4;
1270 		else if (ip_hdr(skb)->version == 6)
1271 			tx_crq.v1.flags1 |= IBMVNIC_TX_PROT_IPV6;
1272 
1273 		if (ip_hdr(skb)->protocol == IPPROTO_TCP)
1274 			tx_crq.v1.flags1 |= IBMVNIC_TX_PROT_TCP;
1275 		else if (ip_hdr(skb)->protocol != IPPROTO_TCP)
1276 			tx_crq.v1.flags1 |= IBMVNIC_TX_PROT_UDP;
1277 	}
1278 
1279 	if (skb->ip_summed == CHECKSUM_PARTIAL) {
1280 		tx_crq.v1.flags1 |= IBMVNIC_TX_CHKSUM_OFFLOAD;
1281 		hdrs += 2;
1282 	}
1283 	/* determine if l2/3/4 headers are sent to firmware */
1284 	if ((*hdrs >> 7) & 1 &&
1285 	    (skb->protocol == htons(ETH_P_IP) ||
1286 	     skb->protocol == htons(ETH_P_IPV6))) {
1287 		build_hdr_descs_arr(tx_buff, &num_entries, *hdrs);
1288 		tx_crq.v1.n_crq_elem = num_entries;
1289 		tx_buff->indir_arr[0] = tx_crq;
1290 		tx_buff->indir_dma = dma_map_single(dev, tx_buff->indir_arr,
1291 						    sizeof(tx_buff->indir_arr),
1292 						    DMA_TO_DEVICE);
1293 		if (dma_mapping_error(dev, tx_buff->indir_dma)) {
1294 			dev_kfree_skb_any(skb);
1295 			tx_buff->skb = NULL;
1296 			if (!firmware_has_feature(FW_FEATURE_CMO))
1297 				dev_err(dev, "tx: unable to map descriptor array\n");
1298 			tx_map_failed++;
1299 			tx_dropped++;
1300 			ret = NETDEV_TX_OK;
1301 			goto out;
1302 		}
1303 		lpar_rc = send_subcrq_indirect(adapter, handle_array[queue_num],
1304 					       (u64)tx_buff->indir_dma,
1305 					       (u64)num_entries);
1306 	} else {
1307 		lpar_rc = send_subcrq(adapter, handle_array[queue_num],
1308 				      &tx_crq);
1309 	}
1310 	if (lpar_rc != H_SUCCESS) {
1311 		dev_err(dev, "tx failed with code %ld\n", lpar_rc);
1312 
1313 		if (tx_pool->consumer_index == 0)
1314 			tx_pool->consumer_index =
1315 				adapter->req_tx_entries_per_subcrq - 1;
1316 		else
1317 			tx_pool->consumer_index--;
1318 
1319 		dev_kfree_skb_any(skb);
1320 		tx_buff->skb = NULL;
1321 
1322 		if (lpar_rc == H_CLOSED) {
1323 			/* Disable TX and report carrier off if queue is closed.
1324 			 * Firmware guarantees that a signal will be sent to the
1325 			 * driver, triggering a reset or some other action.
1326 			 */
1327 			netif_tx_stop_all_queues(netdev);
1328 			netif_carrier_off(netdev);
1329 		}
1330 
1331 		tx_send_failed++;
1332 		tx_dropped++;
1333 		ret = NETDEV_TX_OK;
1334 		goto out;
1335 	}
1336 
1337 	if (atomic_inc_return(&tx_scrq->used)
1338 					>= adapter->req_tx_entries_per_subcrq) {
1339 		netdev_info(netdev, "Stopping queue %d\n", queue_num);
1340 		netif_stop_subqueue(netdev, queue_num);
1341 	}
1342 
1343 	tx_packets++;
1344 	tx_bytes += skb->len;
1345 	txq->trans_start = jiffies;
1346 	ret = NETDEV_TX_OK;
1347 
1348 out:
1349 	netdev->stats.tx_dropped += tx_dropped;
1350 	netdev->stats.tx_bytes += tx_bytes;
1351 	netdev->stats.tx_packets += tx_packets;
1352 	adapter->tx_send_failed += tx_send_failed;
1353 	adapter->tx_map_failed += tx_map_failed;
1354 	adapter->tx_stats_buffers[queue_num].packets += tx_packets;
1355 	adapter->tx_stats_buffers[queue_num].bytes += tx_bytes;
1356 	adapter->tx_stats_buffers[queue_num].dropped_packets += tx_dropped;
1357 
1358 	return ret;
1359 }
1360 
ibmvnic_set_multi(struct net_device * netdev)1361 static void ibmvnic_set_multi(struct net_device *netdev)
1362 {
1363 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
1364 	struct netdev_hw_addr *ha;
1365 	union ibmvnic_crq crq;
1366 
1367 	memset(&crq, 0, sizeof(crq));
1368 	crq.request_capability.first = IBMVNIC_CRQ_CMD;
1369 	crq.request_capability.cmd = REQUEST_CAPABILITY;
1370 
1371 	if (netdev->flags & IFF_PROMISC) {
1372 		if (!adapter->promisc_supported)
1373 			return;
1374 	} else {
1375 		if (netdev->flags & IFF_ALLMULTI) {
1376 			/* Accept all multicast */
1377 			memset(&crq, 0, sizeof(crq));
1378 			crq.multicast_ctrl.first = IBMVNIC_CRQ_CMD;
1379 			crq.multicast_ctrl.cmd = MULTICAST_CTRL;
1380 			crq.multicast_ctrl.flags = IBMVNIC_ENABLE_ALL;
1381 			ibmvnic_send_crq(adapter, &crq);
1382 		} else if (netdev_mc_empty(netdev)) {
1383 			/* Reject all multicast */
1384 			memset(&crq, 0, sizeof(crq));
1385 			crq.multicast_ctrl.first = IBMVNIC_CRQ_CMD;
1386 			crq.multicast_ctrl.cmd = MULTICAST_CTRL;
1387 			crq.multicast_ctrl.flags = IBMVNIC_DISABLE_ALL;
1388 			ibmvnic_send_crq(adapter, &crq);
1389 		} else {
1390 			/* Accept one or more multicast(s) */
1391 			netdev_for_each_mc_addr(ha, netdev) {
1392 				memset(&crq, 0, sizeof(crq));
1393 				crq.multicast_ctrl.first = IBMVNIC_CRQ_CMD;
1394 				crq.multicast_ctrl.cmd = MULTICAST_CTRL;
1395 				crq.multicast_ctrl.flags = IBMVNIC_ENABLE_MC;
1396 				ether_addr_copy(&crq.multicast_ctrl.mac_addr[0],
1397 						ha->addr);
1398 				ibmvnic_send_crq(adapter, &crq);
1399 			}
1400 		}
1401 	}
1402 }
1403 
ibmvnic_set_mac(struct net_device * netdev,void * p)1404 static int ibmvnic_set_mac(struct net_device *netdev, void *p)
1405 {
1406 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
1407 	struct sockaddr *addr = p;
1408 	union ibmvnic_crq crq;
1409 
1410 	if (!is_valid_ether_addr(addr->sa_data))
1411 		return -EADDRNOTAVAIL;
1412 
1413 	memset(&crq, 0, sizeof(crq));
1414 	crq.change_mac_addr.first = IBMVNIC_CRQ_CMD;
1415 	crq.change_mac_addr.cmd = CHANGE_MAC_ADDR;
1416 	ether_addr_copy(&crq.change_mac_addr.mac_addr[0], addr->sa_data);
1417 	ibmvnic_send_crq(adapter, &crq);
1418 	/* netdev->dev_addr is changed in handle_change_mac_rsp function */
1419 	return 0;
1420 }
1421 
1422 /**
1423  * do_reset returns zero if we are able to keep processing reset events, or
1424  * non-zero if we hit a fatal error and must halt.
1425  */
do_reset(struct ibmvnic_adapter * adapter,struct ibmvnic_rwi * rwi,u32 reset_state)1426 static int do_reset(struct ibmvnic_adapter *adapter,
1427 		    struct ibmvnic_rwi *rwi, u32 reset_state)
1428 {
1429 	struct net_device *netdev = adapter->netdev;
1430 	int i, rc;
1431 
1432 	netdev_dbg(adapter->netdev, "Re-setting driver (%d)\n",
1433 		   rwi->reset_reason);
1434 
1435 	netif_carrier_off(netdev);
1436 	adapter->reset_reason = rwi->reset_reason;
1437 
1438 	if (rwi->reset_reason == VNIC_RESET_MOBILITY) {
1439 		rc = ibmvnic_reenable_crq_queue(adapter);
1440 		if (rc)
1441 			return 0;
1442 	}
1443 
1444 	rc = __ibmvnic_close(netdev);
1445 	if (rc)
1446 		return rc;
1447 
1448 	if (adapter->reset_reason != VNIC_RESET_NON_FATAL) {
1449 		/* remove the closed state so when we call open it appears
1450 		 * we are coming from the probed state.
1451 		 */
1452 		adapter->state = VNIC_PROBED;
1453 
1454 		rc = ibmvnic_init(adapter);
1455 		if (rc)
1456 			return 0;
1457 
1458 		/* If the adapter was in PROBE state prior to the reset,
1459 		 * exit here.
1460 		 */
1461 		if (reset_state == VNIC_PROBED)
1462 			return 0;
1463 
1464 		rc = ibmvnic_login(netdev);
1465 		if (rc) {
1466 			adapter->state = reset_state;
1467 			return rc;
1468 		}
1469 
1470 		rc = reset_tx_pools(adapter);
1471 		if (rc)
1472 			return rc;
1473 
1474 		rc = reset_rx_pools(adapter);
1475 		if (rc)
1476 			return rc;
1477 
1478 		if (reset_state == VNIC_CLOSED)
1479 			return 0;
1480 	}
1481 
1482 	rc = __ibmvnic_open(netdev);
1483 	if (rc) {
1484 		if (list_empty(&adapter->rwi_list))
1485 			adapter->state = VNIC_CLOSED;
1486 		else
1487 			adapter->state = reset_state;
1488 
1489 		return 0;
1490 	}
1491 
1492 	/* refresh device's multicast list */
1493 	ibmvnic_set_multi(netdev);
1494 
1495 	/* kick napi */
1496 	for (i = 0; i < adapter->req_rx_queues; i++)
1497 		napi_schedule(&adapter->napi[i]);
1498 
1499 	if (adapter->reset_reason != VNIC_RESET_FAILOVER)
1500 		netdev_notify_peers(netdev);
1501 
1502 	netif_carrier_on(netdev);
1503 
1504 	return 0;
1505 }
1506 
get_next_rwi(struct ibmvnic_adapter * adapter)1507 static struct ibmvnic_rwi *get_next_rwi(struct ibmvnic_adapter *adapter)
1508 {
1509 	struct ibmvnic_rwi *rwi;
1510 
1511 	mutex_lock(&adapter->rwi_lock);
1512 
1513 	if (!list_empty(&adapter->rwi_list)) {
1514 		rwi = list_first_entry(&adapter->rwi_list, struct ibmvnic_rwi,
1515 				       list);
1516 		list_del(&rwi->list);
1517 	} else {
1518 		rwi = NULL;
1519 	}
1520 
1521 	mutex_unlock(&adapter->rwi_lock);
1522 	return rwi;
1523 }
1524 
free_all_rwi(struct ibmvnic_adapter * adapter)1525 static void free_all_rwi(struct ibmvnic_adapter *adapter)
1526 {
1527 	struct ibmvnic_rwi *rwi;
1528 
1529 	rwi = get_next_rwi(adapter);
1530 	while (rwi) {
1531 		kfree(rwi);
1532 		rwi = get_next_rwi(adapter);
1533 	}
1534 }
1535 
__ibmvnic_reset(struct work_struct * work)1536 static void __ibmvnic_reset(struct work_struct *work)
1537 {
1538 	struct ibmvnic_rwi *rwi;
1539 	struct ibmvnic_adapter *adapter;
1540 	struct net_device *netdev;
1541 	u32 reset_state;
1542 	int rc;
1543 
1544 	adapter = container_of(work, struct ibmvnic_adapter, ibmvnic_reset);
1545 	netdev = adapter->netdev;
1546 
1547 	mutex_lock(&adapter->reset_lock);
1548 	adapter->resetting = true;
1549 	reset_state = adapter->state;
1550 
1551 	rwi = get_next_rwi(adapter);
1552 	while (rwi) {
1553 		rc = do_reset(adapter, rwi, reset_state);
1554 		kfree(rwi);
1555 		if (rc)
1556 			break;
1557 
1558 		rwi = get_next_rwi(adapter);
1559 	}
1560 
1561 	if (rc) {
1562 		netdev_dbg(adapter->netdev, "Reset failed\n");
1563 		free_all_rwi(adapter);
1564 		mutex_unlock(&adapter->reset_lock);
1565 		return;
1566 	}
1567 
1568 	adapter->resetting = false;
1569 	mutex_unlock(&adapter->reset_lock);
1570 }
1571 
ibmvnic_reset(struct ibmvnic_adapter * adapter,enum ibmvnic_reset_reason reason)1572 static void ibmvnic_reset(struct ibmvnic_adapter *adapter,
1573 			  enum ibmvnic_reset_reason reason)
1574 {
1575 	struct ibmvnic_rwi *rwi, *tmp;
1576 	struct net_device *netdev = adapter->netdev;
1577 	struct list_head *entry;
1578 
1579 	if (adapter->state == VNIC_REMOVING ||
1580 	    adapter->state == VNIC_REMOVED) {
1581 		netdev_dbg(netdev, "Adapter removing, skipping reset\n");
1582 		return;
1583 	}
1584 
1585 	if (adapter->state == VNIC_PROBING) {
1586 		netdev_warn(netdev, "Adapter reset during probe\n");
1587 		adapter->init_done_rc = EAGAIN;
1588 		return;
1589 	}
1590 
1591 	mutex_lock(&adapter->rwi_lock);
1592 
1593 	list_for_each(entry, &adapter->rwi_list) {
1594 		tmp = list_entry(entry, struct ibmvnic_rwi, list);
1595 		if (tmp->reset_reason == reason) {
1596 			netdev_dbg(netdev, "Skipping matching reset\n");
1597 			mutex_unlock(&adapter->rwi_lock);
1598 			return;
1599 		}
1600 	}
1601 
1602 	rwi = kzalloc(sizeof(*rwi), GFP_ATOMIC);
1603 	if (!rwi) {
1604 		mutex_unlock(&adapter->rwi_lock);
1605 		ibmvnic_close(netdev);
1606 		return;
1607 	}
1608 
1609 	rwi->reset_reason = reason;
1610 	list_add_tail(&rwi->list, &adapter->rwi_list);
1611 	mutex_unlock(&adapter->rwi_lock);
1612 
1613 	netdev_dbg(adapter->netdev, "Scheduling reset (reason %d)\n", reason);
1614 	schedule_work(&adapter->ibmvnic_reset);
1615 }
1616 
ibmvnic_tx_timeout(struct net_device * dev)1617 static void ibmvnic_tx_timeout(struct net_device *dev)
1618 {
1619 	struct ibmvnic_adapter *adapter = netdev_priv(dev);
1620 
1621 	ibmvnic_reset(adapter, VNIC_RESET_TIMEOUT);
1622 }
1623 
remove_buff_from_pool(struct ibmvnic_adapter * adapter,struct ibmvnic_rx_buff * rx_buff)1624 static void remove_buff_from_pool(struct ibmvnic_adapter *adapter,
1625 				  struct ibmvnic_rx_buff *rx_buff)
1626 {
1627 	struct ibmvnic_rx_pool *pool = &adapter->rx_pool[rx_buff->pool_index];
1628 
1629 	rx_buff->skb = NULL;
1630 
1631 	pool->free_map[pool->next_alloc] = (int)(rx_buff - pool->rx_buff);
1632 	pool->next_alloc = (pool->next_alloc + 1) % pool->size;
1633 
1634 	atomic_dec(&pool->available);
1635 }
1636 
ibmvnic_poll(struct napi_struct * napi,int budget)1637 static int ibmvnic_poll(struct napi_struct *napi, int budget)
1638 {
1639 	struct net_device *netdev = napi->dev;
1640 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
1641 	int scrq_num = (int)(napi - adapter->napi);
1642 	int frames_processed = 0;
1643 
1644 restart_poll:
1645 	while (frames_processed < budget) {
1646 		struct sk_buff *skb;
1647 		struct ibmvnic_rx_buff *rx_buff;
1648 		union sub_crq *next;
1649 		u32 length;
1650 		u16 offset;
1651 		u8 flags = 0;
1652 
1653 		if (unlikely(adapter->resetting)) {
1654 			enable_scrq_irq(adapter, adapter->rx_scrq[scrq_num]);
1655 			napi_complete_done(napi, frames_processed);
1656 			return frames_processed;
1657 		}
1658 
1659 		if (!pending_scrq(adapter, adapter->rx_scrq[scrq_num]))
1660 			break;
1661 		next = ibmvnic_next_scrq(adapter, adapter->rx_scrq[scrq_num]);
1662 		rx_buff =
1663 		    (struct ibmvnic_rx_buff *)be64_to_cpu(next->
1664 							  rx_comp.correlator);
1665 		/* do error checking */
1666 		if (next->rx_comp.rc) {
1667 			netdev_dbg(netdev, "rx buffer returned with rc %x\n",
1668 				   be16_to_cpu(next->rx_comp.rc));
1669 			/* free the entry */
1670 			next->rx_comp.first = 0;
1671 			dev_kfree_skb_any(rx_buff->skb);
1672 			remove_buff_from_pool(adapter, rx_buff);
1673 			continue;
1674 		} else if (!rx_buff->skb) {
1675 			/* free the entry */
1676 			next->rx_comp.first = 0;
1677 			remove_buff_from_pool(adapter, rx_buff);
1678 			continue;
1679 		}
1680 
1681 		length = be32_to_cpu(next->rx_comp.len);
1682 		offset = be16_to_cpu(next->rx_comp.off_frame_data);
1683 		flags = next->rx_comp.flags;
1684 		skb = rx_buff->skb;
1685 		skb_copy_to_linear_data(skb, rx_buff->data + offset,
1686 					length);
1687 
1688 		/* VLAN Header has been stripped by the system firmware and
1689 		 * needs to be inserted by the driver
1690 		 */
1691 		if (adapter->rx_vlan_header_insertion &&
1692 		    (flags & IBMVNIC_VLAN_STRIPPED))
1693 			__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
1694 					       ntohs(next->rx_comp.vlan_tci));
1695 
1696 		/* free the entry */
1697 		next->rx_comp.first = 0;
1698 		remove_buff_from_pool(adapter, rx_buff);
1699 
1700 		skb_put(skb, length);
1701 		skb->protocol = eth_type_trans(skb, netdev);
1702 		skb_record_rx_queue(skb, scrq_num);
1703 
1704 		if (flags & IBMVNIC_IP_CHKSUM_GOOD &&
1705 		    flags & IBMVNIC_TCP_UDP_CHKSUM_GOOD) {
1706 			skb->ip_summed = CHECKSUM_UNNECESSARY;
1707 		}
1708 
1709 		length = skb->len;
1710 		napi_gro_receive(napi, skb); /* send it up */
1711 		netdev->stats.rx_packets++;
1712 		netdev->stats.rx_bytes += length;
1713 		adapter->rx_stats_buffers[scrq_num].packets++;
1714 		adapter->rx_stats_buffers[scrq_num].bytes += length;
1715 		frames_processed++;
1716 	}
1717 
1718 	if (adapter->state != VNIC_CLOSING)
1719 		replenish_rx_pool(adapter, &adapter->rx_pool[scrq_num]);
1720 
1721 	if (frames_processed < budget) {
1722 		enable_scrq_irq(adapter, adapter->rx_scrq[scrq_num]);
1723 		napi_complete_done(napi, frames_processed);
1724 		if (pending_scrq(adapter, adapter->rx_scrq[scrq_num]) &&
1725 		    napi_reschedule(napi)) {
1726 			disable_scrq_irq(adapter, adapter->rx_scrq[scrq_num]);
1727 			goto restart_poll;
1728 		}
1729 	}
1730 	return frames_processed;
1731 }
1732 
1733 #ifdef CONFIG_NET_POLL_CONTROLLER
ibmvnic_netpoll_controller(struct net_device * dev)1734 static void ibmvnic_netpoll_controller(struct net_device *dev)
1735 {
1736 	struct ibmvnic_adapter *adapter = netdev_priv(dev);
1737 	int i;
1738 
1739 	replenish_pools(netdev_priv(dev));
1740 	for (i = 0; i < adapter->req_rx_queues; i++)
1741 		ibmvnic_interrupt_rx(adapter->rx_scrq[i]->irq,
1742 				     adapter->rx_scrq[i]);
1743 }
1744 #endif
1745 
ibmvnic_change_mtu(struct net_device * netdev,int new_mtu)1746 static int ibmvnic_change_mtu(struct net_device *netdev, int new_mtu)
1747 {
1748 	return -EOPNOTSUPP;
1749 }
1750 
1751 static const struct net_device_ops ibmvnic_netdev_ops = {
1752 	.ndo_open		= ibmvnic_open,
1753 	.ndo_stop		= ibmvnic_close,
1754 	.ndo_start_xmit		= ibmvnic_xmit,
1755 	.ndo_set_rx_mode	= ibmvnic_set_multi,
1756 	.ndo_set_mac_address	= ibmvnic_set_mac,
1757 	.ndo_validate_addr	= eth_validate_addr,
1758 	.ndo_tx_timeout		= ibmvnic_tx_timeout,
1759 #ifdef CONFIG_NET_POLL_CONTROLLER
1760 	.ndo_poll_controller	= ibmvnic_netpoll_controller,
1761 #endif
1762 	.ndo_change_mtu		= ibmvnic_change_mtu,
1763 };
1764 
1765 /* ethtool functions */
1766 
ibmvnic_get_link_ksettings(struct net_device * netdev,struct ethtool_link_ksettings * cmd)1767 static int ibmvnic_get_link_ksettings(struct net_device *netdev,
1768 				      struct ethtool_link_ksettings *cmd)
1769 {
1770 	u32 supported, advertising;
1771 
1772 	supported = (SUPPORTED_1000baseT_Full | SUPPORTED_Autoneg |
1773 			  SUPPORTED_FIBRE);
1774 	advertising = (ADVERTISED_1000baseT_Full | ADVERTISED_Autoneg |
1775 			    ADVERTISED_FIBRE);
1776 	cmd->base.speed = SPEED_1000;
1777 	cmd->base.duplex = DUPLEX_FULL;
1778 	cmd->base.port = PORT_FIBRE;
1779 	cmd->base.phy_address = 0;
1780 	cmd->base.autoneg = AUTONEG_ENABLE;
1781 
1782 	ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.supported,
1783 						supported);
1784 	ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.advertising,
1785 						advertising);
1786 
1787 	return 0;
1788 }
1789 
ibmvnic_get_drvinfo(struct net_device * dev,struct ethtool_drvinfo * info)1790 static void ibmvnic_get_drvinfo(struct net_device *dev,
1791 				struct ethtool_drvinfo *info)
1792 {
1793 	strlcpy(info->driver, ibmvnic_driver_name, sizeof(info->driver));
1794 	strlcpy(info->version, IBMVNIC_DRIVER_VERSION, sizeof(info->version));
1795 }
1796 
ibmvnic_get_msglevel(struct net_device * netdev)1797 static u32 ibmvnic_get_msglevel(struct net_device *netdev)
1798 {
1799 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
1800 
1801 	return adapter->msg_enable;
1802 }
1803 
ibmvnic_set_msglevel(struct net_device * netdev,u32 data)1804 static void ibmvnic_set_msglevel(struct net_device *netdev, u32 data)
1805 {
1806 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
1807 
1808 	adapter->msg_enable = data;
1809 }
1810 
ibmvnic_get_link(struct net_device * netdev)1811 static u32 ibmvnic_get_link(struct net_device *netdev)
1812 {
1813 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
1814 
1815 	/* Don't need to send a query because we request a logical link up at
1816 	 * init and then we wait for link state indications
1817 	 */
1818 	return adapter->logical_link_state;
1819 }
1820 
ibmvnic_get_ringparam(struct net_device * netdev,struct ethtool_ringparam * ring)1821 static void ibmvnic_get_ringparam(struct net_device *netdev,
1822 				  struct ethtool_ringparam *ring)
1823 {
1824 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
1825 
1826 	ring->rx_max_pending = adapter->max_rx_add_entries_per_subcrq;
1827 	ring->tx_max_pending = adapter->max_tx_entries_per_subcrq;
1828 	ring->rx_mini_max_pending = 0;
1829 	ring->rx_jumbo_max_pending = 0;
1830 	ring->rx_pending = adapter->req_rx_add_entries_per_subcrq;
1831 	ring->tx_pending = adapter->req_tx_entries_per_subcrq;
1832 	ring->rx_mini_pending = 0;
1833 	ring->rx_jumbo_pending = 0;
1834 }
1835 
ibmvnic_get_channels(struct net_device * netdev,struct ethtool_channels * channels)1836 static void ibmvnic_get_channels(struct net_device *netdev,
1837 				 struct ethtool_channels *channels)
1838 {
1839 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
1840 
1841 	channels->max_rx = adapter->max_rx_queues;
1842 	channels->max_tx = adapter->max_tx_queues;
1843 	channels->max_other = 0;
1844 	channels->max_combined = 0;
1845 	channels->rx_count = adapter->req_rx_queues;
1846 	channels->tx_count = adapter->req_tx_queues;
1847 	channels->other_count = 0;
1848 	channels->combined_count = 0;
1849 }
1850 
ibmvnic_get_strings(struct net_device * dev,u32 stringset,u8 * data)1851 static void ibmvnic_get_strings(struct net_device *dev, u32 stringset, u8 *data)
1852 {
1853 	struct ibmvnic_adapter *adapter = netdev_priv(dev);
1854 	int i;
1855 
1856 	if (stringset != ETH_SS_STATS)
1857 		return;
1858 
1859 	for (i = 0; i < ARRAY_SIZE(ibmvnic_stats); i++, data += ETH_GSTRING_LEN)
1860 		memcpy(data, ibmvnic_stats[i].name, ETH_GSTRING_LEN);
1861 
1862 	for (i = 0; i < adapter->req_tx_queues; i++) {
1863 		snprintf(data, ETH_GSTRING_LEN, "tx%d_packets", i);
1864 		data += ETH_GSTRING_LEN;
1865 
1866 		snprintf(data, ETH_GSTRING_LEN, "tx%d_bytes", i);
1867 		data += ETH_GSTRING_LEN;
1868 
1869 		snprintf(data, ETH_GSTRING_LEN, "tx%d_dropped_packets", i);
1870 		data += ETH_GSTRING_LEN;
1871 	}
1872 
1873 	for (i = 0; i < adapter->req_rx_queues; i++) {
1874 		snprintf(data, ETH_GSTRING_LEN, "rx%d_packets", i);
1875 		data += ETH_GSTRING_LEN;
1876 
1877 		snprintf(data, ETH_GSTRING_LEN, "rx%d_bytes", i);
1878 		data += ETH_GSTRING_LEN;
1879 
1880 		snprintf(data, ETH_GSTRING_LEN, "rx%d_interrupts", i);
1881 		data += ETH_GSTRING_LEN;
1882 	}
1883 }
1884 
ibmvnic_get_sset_count(struct net_device * dev,int sset)1885 static int ibmvnic_get_sset_count(struct net_device *dev, int sset)
1886 {
1887 	struct ibmvnic_adapter *adapter = netdev_priv(dev);
1888 
1889 	switch (sset) {
1890 	case ETH_SS_STATS:
1891 		return ARRAY_SIZE(ibmvnic_stats) +
1892 		       adapter->req_tx_queues * NUM_TX_STATS +
1893 		       adapter->req_rx_queues * NUM_RX_STATS;
1894 	default:
1895 		return -EOPNOTSUPP;
1896 	}
1897 }
1898 
ibmvnic_get_ethtool_stats(struct net_device * dev,struct ethtool_stats * stats,u64 * data)1899 static void ibmvnic_get_ethtool_stats(struct net_device *dev,
1900 				      struct ethtool_stats *stats, u64 *data)
1901 {
1902 	struct ibmvnic_adapter *adapter = netdev_priv(dev);
1903 	union ibmvnic_crq crq;
1904 	int i, j;
1905 
1906 	memset(&crq, 0, sizeof(crq));
1907 	crq.request_statistics.first = IBMVNIC_CRQ_CMD;
1908 	crq.request_statistics.cmd = REQUEST_STATISTICS;
1909 	crq.request_statistics.ioba = cpu_to_be32(adapter->stats_token);
1910 	crq.request_statistics.len =
1911 	    cpu_to_be32(sizeof(struct ibmvnic_statistics));
1912 
1913 	/* Wait for data to be written */
1914 	init_completion(&adapter->stats_done);
1915 	ibmvnic_send_crq(adapter, &crq);
1916 	wait_for_completion(&adapter->stats_done);
1917 
1918 	for (i = 0; i < ARRAY_SIZE(ibmvnic_stats); i++)
1919 		data[i] = be64_to_cpu(IBMVNIC_GET_STAT(adapter,
1920 						ibmvnic_stats[i].offset));
1921 
1922 	for (j = 0; j < adapter->req_tx_queues; j++) {
1923 		data[i] = adapter->tx_stats_buffers[j].packets;
1924 		i++;
1925 		data[i] = adapter->tx_stats_buffers[j].bytes;
1926 		i++;
1927 		data[i] = adapter->tx_stats_buffers[j].dropped_packets;
1928 		i++;
1929 	}
1930 
1931 	for (j = 0; j < adapter->req_rx_queues; j++) {
1932 		data[i] = adapter->rx_stats_buffers[j].packets;
1933 		i++;
1934 		data[i] = adapter->rx_stats_buffers[j].bytes;
1935 		i++;
1936 		data[i] = adapter->rx_stats_buffers[j].interrupts;
1937 		i++;
1938 	}
1939 }
1940 
1941 static const struct ethtool_ops ibmvnic_ethtool_ops = {
1942 	.get_drvinfo		= ibmvnic_get_drvinfo,
1943 	.get_msglevel		= ibmvnic_get_msglevel,
1944 	.set_msglevel		= ibmvnic_set_msglevel,
1945 	.get_link		= ibmvnic_get_link,
1946 	.get_ringparam		= ibmvnic_get_ringparam,
1947 	.get_channels		= ibmvnic_get_channels,
1948 	.get_strings            = ibmvnic_get_strings,
1949 	.get_sset_count         = ibmvnic_get_sset_count,
1950 	.get_ethtool_stats	= ibmvnic_get_ethtool_stats,
1951 	.get_link_ksettings	= ibmvnic_get_link_ksettings,
1952 };
1953 
1954 /* Routines for managing CRQs/sCRQs  */
1955 
reset_one_sub_crq_queue(struct ibmvnic_adapter * adapter,struct ibmvnic_sub_crq_queue * scrq)1956 static int reset_one_sub_crq_queue(struct ibmvnic_adapter *adapter,
1957 				   struct ibmvnic_sub_crq_queue *scrq)
1958 {
1959 	int rc;
1960 
1961 	if (scrq->irq) {
1962 		free_irq(scrq->irq, scrq);
1963 		irq_dispose_mapping(scrq->irq);
1964 		scrq->irq = 0;
1965 	}
1966 
1967 	memset(scrq->msgs, 0, 4 * PAGE_SIZE);
1968 	atomic_set(&scrq->used, 0);
1969 	scrq->cur = 0;
1970 
1971 	rc = h_reg_sub_crq(adapter->vdev->unit_address, scrq->msg_token,
1972 			   4 * PAGE_SIZE, &scrq->crq_num, &scrq->hw_irq);
1973 	return rc;
1974 }
1975 
reset_sub_crq_queues(struct ibmvnic_adapter * adapter)1976 static int reset_sub_crq_queues(struct ibmvnic_adapter *adapter)
1977 {
1978 	int i, rc;
1979 
1980 	for (i = 0; i < adapter->req_tx_queues; i++) {
1981 		netdev_dbg(adapter->netdev, "Re-setting tx_scrq[%d]\n", i);
1982 		rc = reset_one_sub_crq_queue(adapter, adapter->tx_scrq[i]);
1983 		if (rc)
1984 			return rc;
1985 	}
1986 
1987 	for (i = 0; i < adapter->req_rx_queues; i++) {
1988 		netdev_dbg(adapter->netdev, "Re-setting rx_scrq[%d]\n", i);
1989 		rc = reset_one_sub_crq_queue(adapter, adapter->rx_scrq[i]);
1990 		if (rc)
1991 			return rc;
1992 	}
1993 
1994 	return rc;
1995 }
1996 
release_sub_crq_queue(struct ibmvnic_adapter * adapter,struct ibmvnic_sub_crq_queue * scrq)1997 static void release_sub_crq_queue(struct ibmvnic_adapter *adapter,
1998 				  struct ibmvnic_sub_crq_queue *scrq)
1999 {
2000 	struct device *dev = &adapter->vdev->dev;
2001 	long rc;
2002 
2003 	netdev_dbg(adapter->netdev, "Releasing sub-CRQ\n");
2004 
2005 	/* Close the sub-crqs */
2006 	do {
2007 		rc = plpar_hcall_norets(H_FREE_SUB_CRQ,
2008 					adapter->vdev->unit_address,
2009 					scrq->crq_num);
2010 	} while (rc == H_BUSY || H_IS_LONG_BUSY(rc));
2011 
2012 	if (rc) {
2013 		netdev_err(adapter->netdev,
2014 			   "Failed to release sub-CRQ %16lx, rc = %ld\n",
2015 			   scrq->crq_num, rc);
2016 	}
2017 
2018 	dma_unmap_single(dev, scrq->msg_token, 4 * PAGE_SIZE,
2019 			 DMA_BIDIRECTIONAL);
2020 	free_pages((unsigned long)scrq->msgs, 2);
2021 	kfree(scrq);
2022 }
2023 
init_sub_crq_queue(struct ibmvnic_adapter * adapter)2024 static struct ibmvnic_sub_crq_queue *init_sub_crq_queue(struct ibmvnic_adapter
2025 							*adapter)
2026 {
2027 	struct device *dev = &adapter->vdev->dev;
2028 	struct ibmvnic_sub_crq_queue *scrq;
2029 	int rc;
2030 
2031 	scrq = kzalloc(sizeof(*scrq), GFP_KERNEL);
2032 	if (!scrq)
2033 		return NULL;
2034 
2035 	scrq->msgs =
2036 		(union sub_crq *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, 2);
2037 	if (!scrq->msgs) {
2038 		dev_warn(dev, "Couldn't allocate crq queue messages page\n");
2039 		goto zero_page_failed;
2040 	}
2041 
2042 	scrq->msg_token = dma_map_single(dev, scrq->msgs, 4 * PAGE_SIZE,
2043 					 DMA_BIDIRECTIONAL);
2044 	if (dma_mapping_error(dev, scrq->msg_token)) {
2045 		dev_warn(dev, "Couldn't map crq queue messages page\n");
2046 		goto map_failed;
2047 	}
2048 
2049 	rc = h_reg_sub_crq(adapter->vdev->unit_address, scrq->msg_token,
2050 			   4 * PAGE_SIZE, &scrq->crq_num, &scrq->hw_irq);
2051 
2052 	if (rc == H_RESOURCE)
2053 		rc = ibmvnic_reset_crq(adapter);
2054 
2055 	if (rc == H_CLOSED) {
2056 		dev_warn(dev, "Partner adapter not ready, waiting.\n");
2057 	} else if (rc) {
2058 		dev_warn(dev, "Error %d registering sub-crq\n", rc);
2059 		goto reg_failed;
2060 	}
2061 
2062 	scrq->adapter = adapter;
2063 	scrq->size = 4 * PAGE_SIZE / sizeof(*scrq->msgs);
2064 	spin_lock_init(&scrq->lock);
2065 
2066 	netdev_dbg(adapter->netdev,
2067 		   "sub-crq initialized, num %lx, hw_irq=%lx, irq=%x\n",
2068 		   scrq->crq_num, scrq->hw_irq, scrq->irq);
2069 
2070 	return scrq;
2071 
2072 reg_failed:
2073 	dma_unmap_single(dev, scrq->msg_token, 4 * PAGE_SIZE,
2074 			 DMA_BIDIRECTIONAL);
2075 map_failed:
2076 	free_pages((unsigned long)scrq->msgs, 2);
2077 zero_page_failed:
2078 	kfree(scrq);
2079 
2080 	return NULL;
2081 }
2082 
release_sub_crqs(struct ibmvnic_adapter * adapter)2083 static void release_sub_crqs(struct ibmvnic_adapter *adapter)
2084 {
2085 	int i;
2086 
2087 	if (adapter->tx_scrq) {
2088 		for (i = 0; i < adapter->req_tx_queues; i++) {
2089 			if (!adapter->tx_scrq[i])
2090 				continue;
2091 
2092 			netdev_dbg(adapter->netdev, "Releasing tx_scrq[%d]\n",
2093 				   i);
2094 			if (adapter->tx_scrq[i]->irq) {
2095 				free_irq(adapter->tx_scrq[i]->irq,
2096 					 adapter->tx_scrq[i]);
2097 				irq_dispose_mapping(adapter->tx_scrq[i]->irq);
2098 				adapter->tx_scrq[i]->irq = 0;
2099 			}
2100 
2101 			release_sub_crq_queue(adapter, adapter->tx_scrq[i]);
2102 		}
2103 
2104 		kfree(adapter->tx_scrq);
2105 		adapter->tx_scrq = NULL;
2106 	}
2107 
2108 	if (adapter->rx_scrq) {
2109 		for (i = 0; i < adapter->req_rx_queues; i++) {
2110 			if (!adapter->rx_scrq[i])
2111 				continue;
2112 
2113 			netdev_dbg(adapter->netdev, "Releasing rx_scrq[%d]\n",
2114 				   i);
2115 			if (adapter->rx_scrq[i]->irq) {
2116 				free_irq(adapter->rx_scrq[i]->irq,
2117 					 adapter->rx_scrq[i]);
2118 				irq_dispose_mapping(adapter->rx_scrq[i]->irq);
2119 				adapter->rx_scrq[i]->irq = 0;
2120 			}
2121 
2122 			release_sub_crq_queue(adapter, adapter->rx_scrq[i]);
2123 		}
2124 
2125 		kfree(adapter->rx_scrq);
2126 		adapter->rx_scrq = NULL;
2127 	}
2128 }
2129 
disable_scrq_irq(struct ibmvnic_adapter * adapter,struct ibmvnic_sub_crq_queue * scrq)2130 static int disable_scrq_irq(struct ibmvnic_adapter *adapter,
2131 			    struct ibmvnic_sub_crq_queue *scrq)
2132 {
2133 	struct device *dev = &adapter->vdev->dev;
2134 	unsigned long rc;
2135 
2136 	rc = plpar_hcall_norets(H_VIOCTL, adapter->vdev->unit_address,
2137 				H_DISABLE_VIO_INTERRUPT, scrq->hw_irq, 0, 0);
2138 	if (rc)
2139 		dev_err(dev, "Couldn't disable scrq irq 0x%lx. rc=%ld\n",
2140 			scrq->hw_irq, rc);
2141 	return rc;
2142 }
2143 
enable_scrq_irq(struct ibmvnic_adapter * adapter,struct ibmvnic_sub_crq_queue * scrq)2144 static int enable_scrq_irq(struct ibmvnic_adapter *adapter,
2145 			   struct ibmvnic_sub_crq_queue *scrq)
2146 {
2147 	struct device *dev = &adapter->vdev->dev;
2148 	unsigned long rc;
2149 
2150 	if (scrq->hw_irq > 0x100000000ULL) {
2151 		dev_err(dev, "bad hw_irq = %lx\n", scrq->hw_irq);
2152 		return 1;
2153 	}
2154 
2155 	rc = plpar_hcall_norets(H_VIOCTL, adapter->vdev->unit_address,
2156 				H_ENABLE_VIO_INTERRUPT, scrq->hw_irq, 0, 0);
2157 	if (rc)
2158 		dev_err(dev, "Couldn't enable scrq irq 0x%lx. rc=%ld\n",
2159 			scrq->hw_irq, rc);
2160 	return rc;
2161 }
2162 
ibmvnic_complete_tx(struct ibmvnic_adapter * adapter,struct ibmvnic_sub_crq_queue * scrq)2163 static int ibmvnic_complete_tx(struct ibmvnic_adapter *adapter,
2164 			       struct ibmvnic_sub_crq_queue *scrq)
2165 {
2166 	struct device *dev = &adapter->vdev->dev;
2167 	struct ibmvnic_tx_buff *txbuff;
2168 	union sub_crq *next;
2169 	int index;
2170 	int i, j;
2171 	u8 first;
2172 
2173 restart_loop:
2174 	while (pending_scrq(adapter, scrq)) {
2175 		unsigned int pool = scrq->pool_index;
2176 
2177 		next = ibmvnic_next_scrq(adapter, scrq);
2178 		for (i = 0; i < next->tx_comp.num_comps; i++) {
2179 			if (next->tx_comp.rcs[i]) {
2180 				dev_err(dev, "tx error %x\n",
2181 					next->tx_comp.rcs[i]);
2182 				continue;
2183 			}
2184 			index = be32_to_cpu(next->tx_comp.correlators[i]);
2185 			txbuff = &adapter->tx_pool[pool].tx_buff[index];
2186 
2187 			for (j = 0; j < IBMVNIC_MAX_FRAGS_PER_CRQ; j++) {
2188 				if (!txbuff->data_dma[j])
2189 					continue;
2190 
2191 				txbuff->data_dma[j] = 0;
2192 			}
2193 			/* if sub_crq was sent indirectly */
2194 			first = txbuff->indir_arr[0].generic.first;
2195 			if (first == IBMVNIC_CRQ_CMD) {
2196 				dma_unmap_single(dev, txbuff->indir_dma,
2197 						 sizeof(txbuff->indir_arr),
2198 						 DMA_TO_DEVICE);
2199 			}
2200 
2201 			if (txbuff->last_frag) {
2202 				dev_kfree_skb_any(txbuff->skb);
2203 				txbuff->skb = NULL;
2204 			}
2205 
2206 			adapter->tx_pool[pool].free_map[adapter->tx_pool[pool].
2207 						     producer_index] = index;
2208 			adapter->tx_pool[pool].producer_index =
2209 			    (adapter->tx_pool[pool].producer_index + 1) %
2210 			    adapter->req_tx_entries_per_subcrq;
2211 		}
2212 		/* remove tx_comp scrq*/
2213 		next->tx_comp.first = 0;
2214 
2215 		if (atomic_sub_return(next->tx_comp.num_comps, &scrq->used) <=
2216 		    (adapter->req_tx_entries_per_subcrq / 2) &&
2217 		    __netif_subqueue_stopped(adapter->netdev,
2218 					     scrq->pool_index)) {
2219 			netif_wake_subqueue(adapter->netdev, scrq->pool_index);
2220 			netdev_info(adapter->netdev, "Started queue %d\n",
2221 				    scrq->pool_index);
2222 		}
2223 	}
2224 
2225 	enable_scrq_irq(adapter, scrq);
2226 
2227 	if (pending_scrq(adapter, scrq)) {
2228 		disable_scrq_irq(adapter, scrq);
2229 		goto restart_loop;
2230 	}
2231 
2232 	return 0;
2233 }
2234 
ibmvnic_interrupt_tx(int irq,void * instance)2235 static irqreturn_t ibmvnic_interrupt_tx(int irq, void *instance)
2236 {
2237 	struct ibmvnic_sub_crq_queue *scrq = instance;
2238 	struct ibmvnic_adapter *adapter = scrq->adapter;
2239 
2240 	disable_scrq_irq(adapter, scrq);
2241 	ibmvnic_complete_tx(adapter, scrq);
2242 
2243 	return IRQ_HANDLED;
2244 }
2245 
ibmvnic_interrupt_rx(int irq,void * instance)2246 static irqreturn_t ibmvnic_interrupt_rx(int irq, void *instance)
2247 {
2248 	struct ibmvnic_sub_crq_queue *scrq = instance;
2249 	struct ibmvnic_adapter *adapter = scrq->adapter;
2250 
2251 	/* When booting a kdump kernel we can hit pending interrupts
2252 	 * prior to completing driver initialization.
2253 	 */
2254 	if (unlikely(adapter->state != VNIC_OPEN))
2255 		return IRQ_NONE;
2256 
2257 	adapter->rx_stats_buffers[scrq->scrq_num].interrupts++;
2258 
2259 	if (napi_schedule_prep(&adapter->napi[scrq->scrq_num])) {
2260 		disable_scrq_irq(adapter, scrq);
2261 		__napi_schedule(&adapter->napi[scrq->scrq_num]);
2262 	}
2263 
2264 	return IRQ_HANDLED;
2265 }
2266 
init_sub_crq_irqs(struct ibmvnic_adapter * adapter)2267 static int init_sub_crq_irqs(struct ibmvnic_adapter *adapter)
2268 {
2269 	struct device *dev = &adapter->vdev->dev;
2270 	struct ibmvnic_sub_crq_queue *scrq;
2271 	int i = 0, j = 0;
2272 	int rc = 0;
2273 
2274 	for (i = 0; i < adapter->req_tx_queues; i++) {
2275 		netdev_dbg(adapter->netdev, "Initializing tx_scrq[%d] irq\n",
2276 			   i);
2277 		scrq = adapter->tx_scrq[i];
2278 		scrq->irq = irq_create_mapping(NULL, scrq->hw_irq);
2279 
2280 		if (!scrq->irq) {
2281 			rc = -EINVAL;
2282 			dev_err(dev, "Error mapping irq\n");
2283 			goto req_tx_irq_failed;
2284 		}
2285 
2286 		rc = request_irq(scrq->irq, ibmvnic_interrupt_tx,
2287 				 0, "ibmvnic_tx", scrq);
2288 
2289 		if (rc) {
2290 			dev_err(dev, "Couldn't register tx irq 0x%x. rc=%d\n",
2291 				scrq->irq, rc);
2292 			irq_dispose_mapping(scrq->irq);
2293 			goto req_rx_irq_failed;
2294 		}
2295 	}
2296 
2297 	for (i = 0; i < adapter->req_rx_queues; i++) {
2298 		netdev_dbg(adapter->netdev, "Initializing rx_scrq[%d] irq\n",
2299 			   i);
2300 		scrq = adapter->rx_scrq[i];
2301 		scrq->irq = irq_create_mapping(NULL, scrq->hw_irq);
2302 		if (!scrq->irq) {
2303 			rc = -EINVAL;
2304 			dev_err(dev, "Error mapping irq\n");
2305 			goto req_rx_irq_failed;
2306 		}
2307 		rc = request_irq(scrq->irq, ibmvnic_interrupt_rx,
2308 				 0, "ibmvnic_rx", scrq);
2309 		if (rc) {
2310 			dev_err(dev, "Couldn't register rx irq 0x%x. rc=%d\n",
2311 				scrq->irq, rc);
2312 			irq_dispose_mapping(scrq->irq);
2313 			goto req_rx_irq_failed;
2314 		}
2315 	}
2316 	return rc;
2317 
2318 req_rx_irq_failed:
2319 	for (j = 0; j < i; j++) {
2320 		free_irq(adapter->rx_scrq[j]->irq, adapter->rx_scrq[j]);
2321 		irq_dispose_mapping(adapter->rx_scrq[j]->irq);
2322 	}
2323 	i = adapter->req_tx_queues;
2324 req_tx_irq_failed:
2325 	for (j = 0; j < i; j++) {
2326 		free_irq(adapter->tx_scrq[j]->irq, adapter->tx_scrq[j]);
2327 		irq_dispose_mapping(adapter->rx_scrq[j]->irq);
2328 	}
2329 	release_sub_crqs(adapter);
2330 	return rc;
2331 }
2332 
init_sub_crqs(struct ibmvnic_adapter * adapter)2333 static int init_sub_crqs(struct ibmvnic_adapter *adapter)
2334 {
2335 	struct device *dev = &adapter->vdev->dev;
2336 	struct ibmvnic_sub_crq_queue **allqueues;
2337 	int registered_queues = 0;
2338 	int total_queues;
2339 	int more = 0;
2340 	int i;
2341 
2342 	total_queues = adapter->req_tx_queues + adapter->req_rx_queues;
2343 
2344 	allqueues = kcalloc(total_queues, sizeof(*allqueues), GFP_KERNEL);
2345 	if (!allqueues)
2346 		return -1;
2347 
2348 	for (i = 0; i < total_queues; i++) {
2349 		allqueues[i] = init_sub_crq_queue(adapter);
2350 		if (!allqueues[i]) {
2351 			dev_warn(dev, "Couldn't allocate all sub-crqs\n");
2352 			break;
2353 		}
2354 		registered_queues++;
2355 	}
2356 
2357 	/* Make sure we were able to register the minimum number of queues */
2358 	if (registered_queues <
2359 	    adapter->min_tx_queues + adapter->min_rx_queues) {
2360 		dev_err(dev, "Fatal: Couldn't init  min number of sub-crqs\n");
2361 		goto tx_failed;
2362 	}
2363 
2364 	/* Distribute the failed allocated queues*/
2365 	for (i = 0; i < total_queues - registered_queues + more ; i++) {
2366 		netdev_dbg(adapter->netdev, "Reducing number of queues\n");
2367 		switch (i % 3) {
2368 		case 0:
2369 			if (adapter->req_rx_queues > adapter->min_rx_queues)
2370 				adapter->req_rx_queues--;
2371 			else
2372 				more++;
2373 			break;
2374 		case 1:
2375 			if (adapter->req_tx_queues > adapter->min_tx_queues)
2376 				adapter->req_tx_queues--;
2377 			else
2378 				more++;
2379 			break;
2380 		}
2381 	}
2382 
2383 	adapter->tx_scrq = kcalloc(adapter->req_tx_queues,
2384 				   sizeof(*adapter->tx_scrq), GFP_KERNEL);
2385 	if (!adapter->tx_scrq)
2386 		goto tx_failed;
2387 
2388 	for (i = 0; i < adapter->req_tx_queues; i++) {
2389 		adapter->tx_scrq[i] = allqueues[i];
2390 		adapter->tx_scrq[i]->pool_index = i;
2391 	}
2392 
2393 	adapter->rx_scrq = kcalloc(adapter->req_rx_queues,
2394 				   sizeof(*adapter->rx_scrq), GFP_KERNEL);
2395 	if (!adapter->rx_scrq)
2396 		goto rx_failed;
2397 
2398 	for (i = 0; i < adapter->req_rx_queues; i++) {
2399 		adapter->rx_scrq[i] = allqueues[i + adapter->req_tx_queues];
2400 		adapter->rx_scrq[i]->scrq_num = i;
2401 	}
2402 
2403 	kfree(allqueues);
2404 	return 0;
2405 
2406 rx_failed:
2407 	kfree(adapter->tx_scrq);
2408 	adapter->tx_scrq = NULL;
2409 tx_failed:
2410 	for (i = 0; i < registered_queues; i++)
2411 		release_sub_crq_queue(adapter, allqueues[i]);
2412 	kfree(allqueues);
2413 	return -1;
2414 }
2415 
ibmvnic_send_req_caps(struct ibmvnic_adapter * adapter,int retry)2416 static void ibmvnic_send_req_caps(struct ibmvnic_adapter *adapter, int retry)
2417 {
2418 	struct device *dev = &adapter->vdev->dev;
2419 	union ibmvnic_crq crq;
2420 
2421 	if (!retry) {
2422 		/* Sub-CRQ entries are 32 byte long */
2423 		int entries_page = 4 * PAGE_SIZE / (sizeof(u64) * 4);
2424 
2425 		if (adapter->min_tx_entries_per_subcrq > entries_page ||
2426 		    adapter->min_rx_add_entries_per_subcrq > entries_page) {
2427 			dev_err(dev, "Fatal, invalid entries per sub-crq\n");
2428 			return;
2429 		}
2430 
2431 		/* Get the minimum between the queried max and the entries
2432 		 * that fit in our PAGE_SIZE
2433 		 */
2434 		adapter->req_tx_entries_per_subcrq =
2435 		    adapter->max_tx_entries_per_subcrq > entries_page ?
2436 		    entries_page : adapter->max_tx_entries_per_subcrq;
2437 		adapter->req_rx_add_entries_per_subcrq =
2438 		    adapter->max_rx_add_entries_per_subcrq > entries_page ?
2439 		    entries_page : adapter->max_rx_add_entries_per_subcrq;
2440 
2441 		adapter->req_tx_queues = adapter->opt_tx_comp_sub_queues;
2442 		adapter->req_rx_queues = adapter->opt_rx_comp_queues;
2443 		adapter->req_rx_add_queues = adapter->max_rx_add_queues;
2444 
2445 		adapter->req_mtu = adapter->netdev->mtu + ETH_HLEN;
2446 	}
2447 
2448 	memset(&crq, 0, sizeof(crq));
2449 	crq.request_capability.first = IBMVNIC_CRQ_CMD;
2450 	crq.request_capability.cmd = REQUEST_CAPABILITY;
2451 
2452 	crq.request_capability.capability = cpu_to_be16(REQ_TX_QUEUES);
2453 	crq.request_capability.number = cpu_to_be64(adapter->req_tx_queues);
2454 	atomic_inc(&adapter->running_cap_crqs);
2455 	ibmvnic_send_crq(adapter, &crq);
2456 
2457 	crq.request_capability.capability = cpu_to_be16(REQ_RX_QUEUES);
2458 	crq.request_capability.number = cpu_to_be64(adapter->req_rx_queues);
2459 	atomic_inc(&adapter->running_cap_crqs);
2460 	ibmvnic_send_crq(adapter, &crq);
2461 
2462 	crq.request_capability.capability = cpu_to_be16(REQ_RX_ADD_QUEUES);
2463 	crq.request_capability.number = cpu_to_be64(adapter->req_rx_add_queues);
2464 	atomic_inc(&adapter->running_cap_crqs);
2465 	ibmvnic_send_crq(adapter, &crq);
2466 
2467 	crq.request_capability.capability =
2468 	    cpu_to_be16(REQ_TX_ENTRIES_PER_SUBCRQ);
2469 	crq.request_capability.number =
2470 	    cpu_to_be64(adapter->req_tx_entries_per_subcrq);
2471 	atomic_inc(&adapter->running_cap_crqs);
2472 	ibmvnic_send_crq(adapter, &crq);
2473 
2474 	crq.request_capability.capability =
2475 	    cpu_to_be16(REQ_RX_ADD_ENTRIES_PER_SUBCRQ);
2476 	crq.request_capability.number =
2477 	    cpu_to_be64(adapter->req_rx_add_entries_per_subcrq);
2478 	atomic_inc(&adapter->running_cap_crqs);
2479 	ibmvnic_send_crq(adapter, &crq);
2480 
2481 	crq.request_capability.capability = cpu_to_be16(REQ_MTU);
2482 	crq.request_capability.number = cpu_to_be64(adapter->req_mtu);
2483 	atomic_inc(&adapter->running_cap_crqs);
2484 	ibmvnic_send_crq(adapter, &crq);
2485 
2486 	if (adapter->netdev->flags & IFF_PROMISC) {
2487 		if (adapter->promisc_supported) {
2488 			crq.request_capability.capability =
2489 			    cpu_to_be16(PROMISC_REQUESTED);
2490 			crq.request_capability.number = cpu_to_be64(1);
2491 			atomic_inc(&adapter->running_cap_crqs);
2492 			ibmvnic_send_crq(adapter, &crq);
2493 		}
2494 	} else {
2495 		crq.request_capability.capability =
2496 		    cpu_to_be16(PROMISC_REQUESTED);
2497 		crq.request_capability.number = cpu_to_be64(0);
2498 		atomic_inc(&adapter->running_cap_crqs);
2499 		ibmvnic_send_crq(adapter, &crq);
2500 	}
2501 }
2502 
pending_scrq(struct ibmvnic_adapter * adapter,struct ibmvnic_sub_crq_queue * scrq)2503 static int pending_scrq(struct ibmvnic_adapter *adapter,
2504 			struct ibmvnic_sub_crq_queue *scrq)
2505 {
2506 	union sub_crq *entry = &scrq->msgs[scrq->cur];
2507 
2508 	if (entry->generic.first & IBMVNIC_CRQ_CMD_RSP)
2509 		return 1;
2510 	else
2511 		return 0;
2512 }
2513 
ibmvnic_next_scrq(struct ibmvnic_adapter * adapter,struct ibmvnic_sub_crq_queue * scrq)2514 static union sub_crq *ibmvnic_next_scrq(struct ibmvnic_adapter *adapter,
2515 					struct ibmvnic_sub_crq_queue *scrq)
2516 {
2517 	union sub_crq *entry;
2518 	unsigned long flags;
2519 
2520 	spin_lock_irqsave(&scrq->lock, flags);
2521 	entry = &scrq->msgs[scrq->cur];
2522 	if (entry->generic.first & IBMVNIC_CRQ_CMD_RSP) {
2523 		if (++scrq->cur == scrq->size)
2524 			scrq->cur = 0;
2525 	} else {
2526 		entry = NULL;
2527 	}
2528 	spin_unlock_irqrestore(&scrq->lock, flags);
2529 
2530 	return entry;
2531 }
2532 
ibmvnic_next_crq(struct ibmvnic_adapter * adapter)2533 static union ibmvnic_crq *ibmvnic_next_crq(struct ibmvnic_adapter *adapter)
2534 {
2535 	struct ibmvnic_crq_queue *queue = &adapter->crq;
2536 	union ibmvnic_crq *crq;
2537 
2538 	crq = &queue->msgs[queue->cur];
2539 	if (crq->generic.first & IBMVNIC_CRQ_CMD_RSP) {
2540 		if (++queue->cur == queue->size)
2541 			queue->cur = 0;
2542 	} else {
2543 		crq = NULL;
2544 	}
2545 
2546 	return crq;
2547 }
2548 
send_subcrq(struct ibmvnic_adapter * adapter,u64 remote_handle,union sub_crq * sub_crq)2549 static int send_subcrq(struct ibmvnic_adapter *adapter, u64 remote_handle,
2550 		       union sub_crq *sub_crq)
2551 {
2552 	unsigned int ua = adapter->vdev->unit_address;
2553 	struct device *dev = &adapter->vdev->dev;
2554 	u64 *u64_crq = (u64 *)sub_crq;
2555 	int rc;
2556 
2557 	netdev_dbg(adapter->netdev,
2558 		   "Sending sCRQ %016lx: %016lx %016lx %016lx %016lx\n",
2559 		   (unsigned long int)cpu_to_be64(remote_handle),
2560 		   (unsigned long int)cpu_to_be64(u64_crq[0]),
2561 		   (unsigned long int)cpu_to_be64(u64_crq[1]),
2562 		   (unsigned long int)cpu_to_be64(u64_crq[2]),
2563 		   (unsigned long int)cpu_to_be64(u64_crq[3]));
2564 
2565 	/* Make sure the hypervisor sees the complete request */
2566 	mb();
2567 
2568 	rc = plpar_hcall_norets(H_SEND_SUB_CRQ, ua,
2569 				cpu_to_be64(remote_handle),
2570 				cpu_to_be64(u64_crq[0]),
2571 				cpu_to_be64(u64_crq[1]),
2572 				cpu_to_be64(u64_crq[2]),
2573 				cpu_to_be64(u64_crq[3]));
2574 
2575 	if (rc) {
2576 		if (rc == H_CLOSED)
2577 			dev_warn(dev, "CRQ Queue closed\n");
2578 		dev_err(dev, "Send error (rc=%d)\n", rc);
2579 	}
2580 
2581 	return rc;
2582 }
2583 
send_subcrq_indirect(struct ibmvnic_adapter * adapter,u64 remote_handle,u64 ioba,u64 num_entries)2584 static int send_subcrq_indirect(struct ibmvnic_adapter *adapter,
2585 				u64 remote_handle, u64 ioba, u64 num_entries)
2586 {
2587 	unsigned int ua = adapter->vdev->unit_address;
2588 	struct device *dev = &adapter->vdev->dev;
2589 	int rc;
2590 
2591 	/* Make sure the hypervisor sees the complete request */
2592 	mb();
2593 	rc = plpar_hcall_norets(H_SEND_SUB_CRQ_INDIRECT, ua,
2594 				cpu_to_be64(remote_handle),
2595 				ioba, num_entries);
2596 
2597 	if (rc) {
2598 		if (rc == H_CLOSED)
2599 			dev_warn(dev, "CRQ Queue closed\n");
2600 		dev_err(dev, "Send (indirect) error (rc=%d)\n", rc);
2601 	}
2602 
2603 	return rc;
2604 }
2605 
ibmvnic_send_crq(struct ibmvnic_adapter * adapter,union ibmvnic_crq * crq)2606 static int ibmvnic_send_crq(struct ibmvnic_adapter *adapter,
2607 			    union ibmvnic_crq *crq)
2608 {
2609 	unsigned int ua = adapter->vdev->unit_address;
2610 	struct device *dev = &adapter->vdev->dev;
2611 	u64 *u64_crq = (u64 *)crq;
2612 	int rc;
2613 
2614 	netdev_dbg(adapter->netdev, "Sending CRQ: %016lx %016lx\n",
2615 		   (unsigned long int)cpu_to_be64(u64_crq[0]),
2616 		   (unsigned long int)cpu_to_be64(u64_crq[1]));
2617 
2618 	/* Make sure the hypervisor sees the complete request */
2619 	mb();
2620 
2621 	rc = plpar_hcall_norets(H_SEND_CRQ, ua,
2622 				cpu_to_be64(u64_crq[0]),
2623 				cpu_to_be64(u64_crq[1]));
2624 
2625 	if (rc) {
2626 		if (rc == H_CLOSED)
2627 			dev_warn(dev, "CRQ Queue closed\n");
2628 		dev_warn(dev, "Send error (rc=%d)\n", rc);
2629 	}
2630 
2631 	return rc;
2632 }
2633 
ibmvnic_send_crq_init(struct ibmvnic_adapter * adapter)2634 static int ibmvnic_send_crq_init(struct ibmvnic_adapter *adapter)
2635 {
2636 	union ibmvnic_crq crq;
2637 
2638 	memset(&crq, 0, sizeof(crq));
2639 	crq.generic.first = IBMVNIC_CRQ_INIT_CMD;
2640 	crq.generic.cmd = IBMVNIC_CRQ_INIT;
2641 	netdev_dbg(adapter->netdev, "Sending CRQ init\n");
2642 
2643 	return ibmvnic_send_crq(adapter, &crq);
2644 }
2645 
send_version_xchg(struct ibmvnic_adapter * adapter)2646 static int send_version_xchg(struct ibmvnic_adapter *adapter)
2647 {
2648 	union ibmvnic_crq crq;
2649 
2650 	memset(&crq, 0, sizeof(crq));
2651 	crq.version_exchange.first = IBMVNIC_CRQ_CMD;
2652 	crq.version_exchange.cmd = VERSION_EXCHANGE;
2653 	crq.version_exchange.version = cpu_to_be16(ibmvnic_version);
2654 
2655 	return ibmvnic_send_crq(adapter, &crq);
2656 }
2657 
send_login(struct ibmvnic_adapter * adapter)2658 static void send_login(struct ibmvnic_adapter *adapter)
2659 {
2660 	struct ibmvnic_login_rsp_buffer *login_rsp_buffer;
2661 	struct ibmvnic_login_buffer *login_buffer;
2662 	struct device *dev = &adapter->vdev->dev;
2663 	dma_addr_t rsp_buffer_token;
2664 	dma_addr_t buffer_token;
2665 	size_t rsp_buffer_size;
2666 	union ibmvnic_crq crq;
2667 	size_t buffer_size;
2668 	__be64 *tx_list_p;
2669 	__be64 *rx_list_p;
2670 	int i;
2671 
2672 	buffer_size =
2673 	    sizeof(struct ibmvnic_login_buffer) +
2674 	    sizeof(u64) * (adapter->req_tx_queues + adapter->req_rx_queues);
2675 
2676 	login_buffer = kmalloc(buffer_size, GFP_ATOMIC);
2677 	if (!login_buffer)
2678 		goto buf_alloc_failed;
2679 
2680 	buffer_token = dma_map_single(dev, login_buffer, buffer_size,
2681 				      DMA_TO_DEVICE);
2682 	if (dma_mapping_error(dev, buffer_token)) {
2683 		dev_err(dev, "Couldn't map login buffer\n");
2684 		goto buf_map_failed;
2685 	}
2686 
2687 	rsp_buffer_size = sizeof(struct ibmvnic_login_rsp_buffer) +
2688 			  sizeof(u64) * adapter->req_tx_queues +
2689 			  sizeof(u64) * adapter->req_rx_queues +
2690 			  sizeof(u64) * adapter->req_rx_queues +
2691 			  sizeof(u8) * IBMVNIC_TX_DESC_VERSIONS;
2692 
2693 	login_rsp_buffer = kmalloc(rsp_buffer_size, GFP_ATOMIC);
2694 	if (!login_rsp_buffer)
2695 		goto buf_rsp_alloc_failed;
2696 
2697 	rsp_buffer_token = dma_map_single(dev, login_rsp_buffer,
2698 					  rsp_buffer_size, DMA_FROM_DEVICE);
2699 	if (dma_mapping_error(dev, rsp_buffer_token)) {
2700 		dev_err(dev, "Couldn't map login rsp buffer\n");
2701 		goto buf_rsp_map_failed;
2702 	}
2703 
2704 	adapter->login_buf = login_buffer;
2705 	adapter->login_buf_token = buffer_token;
2706 	adapter->login_buf_sz = buffer_size;
2707 	adapter->login_rsp_buf = login_rsp_buffer;
2708 	adapter->login_rsp_buf_token = rsp_buffer_token;
2709 	adapter->login_rsp_buf_sz = rsp_buffer_size;
2710 
2711 	login_buffer->len = cpu_to_be32(buffer_size);
2712 	login_buffer->version = cpu_to_be32(INITIAL_VERSION_LB);
2713 	login_buffer->num_txcomp_subcrqs = cpu_to_be32(adapter->req_tx_queues);
2714 	login_buffer->off_txcomp_subcrqs =
2715 	    cpu_to_be32(sizeof(struct ibmvnic_login_buffer));
2716 	login_buffer->num_rxcomp_subcrqs = cpu_to_be32(adapter->req_rx_queues);
2717 	login_buffer->off_rxcomp_subcrqs =
2718 	    cpu_to_be32(sizeof(struct ibmvnic_login_buffer) +
2719 			sizeof(u64) * adapter->req_tx_queues);
2720 	login_buffer->login_rsp_ioba = cpu_to_be32(rsp_buffer_token);
2721 	login_buffer->login_rsp_len = cpu_to_be32(rsp_buffer_size);
2722 
2723 	tx_list_p = (__be64 *)((char *)login_buffer +
2724 				      sizeof(struct ibmvnic_login_buffer));
2725 	rx_list_p = (__be64 *)((char *)login_buffer +
2726 				      sizeof(struct ibmvnic_login_buffer) +
2727 				      sizeof(u64) * adapter->req_tx_queues);
2728 
2729 	for (i = 0; i < adapter->req_tx_queues; i++) {
2730 		if (adapter->tx_scrq[i]) {
2731 			tx_list_p[i] = cpu_to_be64(adapter->tx_scrq[i]->
2732 						   crq_num);
2733 		}
2734 	}
2735 
2736 	for (i = 0; i < adapter->req_rx_queues; i++) {
2737 		if (adapter->rx_scrq[i]) {
2738 			rx_list_p[i] = cpu_to_be64(adapter->rx_scrq[i]->
2739 						   crq_num);
2740 		}
2741 	}
2742 
2743 	netdev_dbg(adapter->netdev, "Login Buffer:\n");
2744 	for (i = 0; i < (adapter->login_buf_sz - 1) / 8 + 1; i++) {
2745 		netdev_dbg(adapter->netdev, "%016lx\n",
2746 			   ((unsigned long int *)(adapter->login_buf))[i]);
2747 	}
2748 
2749 	memset(&crq, 0, sizeof(crq));
2750 	crq.login.first = IBMVNIC_CRQ_CMD;
2751 	crq.login.cmd = LOGIN;
2752 	crq.login.ioba = cpu_to_be32(buffer_token);
2753 	crq.login.len = cpu_to_be32(buffer_size);
2754 	ibmvnic_send_crq(adapter, &crq);
2755 
2756 	return;
2757 
2758 buf_rsp_map_failed:
2759 	kfree(login_rsp_buffer);
2760 buf_rsp_alloc_failed:
2761 	dma_unmap_single(dev, buffer_token, buffer_size, DMA_TO_DEVICE);
2762 buf_map_failed:
2763 	kfree(login_buffer);
2764 buf_alloc_failed:
2765 	return;
2766 }
2767 
send_request_map(struct ibmvnic_adapter * adapter,dma_addr_t addr,u32 len,u8 map_id)2768 static void send_request_map(struct ibmvnic_adapter *adapter, dma_addr_t addr,
2769 			     u32 len, u8 map_id)
2770 {
2771 	union ibmvnic_crq crq;
2772 
2773 	memset(&crq, 0, sizeof(crq));
2774 	crq.request_map.first = IBMVNIC_CRQ_CMD;
2775 	crq.request_map.cmd = REQUEST_MAP;
2776 	crq.request_map.map_id = map_id;
2777 	crq.request_map.ioba = cpu_to_be32(addr);
2778 	crq.request_map.len = cpu_to_be32(len);
2779 	ibmvnic_send_crq(adapter, &crq);
2780 }
2781 
send_request_unmap(struct ibmvnic_adapter * adapter,u8 map_id)2782 static void send_request_unmap(struct ibmvnic_adapter *adapter, u8 map_id)
2783 {
2784 	union ibmvnic_crq crq;
2785 
2786 	memset(&crq, 0, sizeof(crq));
2787 	crq.request_unmap.first = IBMVNIC_CRQ_CMD;
2788 	crq.request_unmap.cmd = REQUEST_UNMAP;
2789 	crq.request_unmap.map_id = map_id;
2790 	ibmvnic_send_crq(adapter, &crq);
2791 }
2792 
send_map_query(struct ibmvnic_adapter * adapter)2793 static void send_map_query(struct ibmvnic_adapter *adapter)
2794 {
2795 	union ibmvnic_crq crq;
2796 
2797 	memset(&crq, 0, sizeof(crq));
2798 	crq.query_map.first = IBMVNIC_CRQ_CMD;
2799 	crq.query_map.cmd = QUERY_MAP;
2800 	ibmvnic_send_crq(adapter, &crq);
2801 }
2802 
2803 /* Send a series of CRQs requesting various capabilities of the VNIC server */
send_cap_queries(struct ibmvnic_adapter * adapter)2804 static void send_cap_queries(struct ibmvnic_adapter *adapter)
2805 {
2806 	union ibmvnic_crq crq;
2807 
2808 	atomic_set(&adapter->running_cap_crqs, 0);
2809 	memset(&crq, 0, sizeof(crq));
2810 	crq.query_capability.first = IBMVNIC_CRQ_CMD;
2811 	crq.query_capability.cmd = QUERY_CAPABILITY;
2812 
2813 	crq.query_capability.capability = cpu_to_be16(MIN_TX_QUEUES);
2814 	atomic_inc(&adapter->running_cap_crqs);
2815 	ibmvnic_send_crq(adapter, &crq);
2816 
2817 	crq.query_capability.capability = cpu_to_be16(MIN_RX_QUEUES);
2818 	atomic_inc(&adapter->running_cap_crqs);
2819 	ibmvnic_send_crq(adapter, &crq);
2820 
2821 	crq.query_capability.capability = cpu_to_be16(MIN_RX_ADD_QUEUES);
2822 	atomic_inc(&adapter->running_cap_crqs);
2823 	ibmvnic_send_crq(adapter, &crq);
2824 
2825 	crq.query_capability.capability = cpu_to_be16(MAX_TX_QUEUES);
2826 	atomic_inc(&adapter->running_cap_crqs);
2827 	ibmvnic_send_crq(adapter, &crq);
2828 
2829 	crq.query_capability.capability = cpu_to_be16(MAX_RX_QUEUES);
2830 	atomic_inc(&adapter->running_cap_crqs);
2831 	ibmvnic_send_crq(adapter, &crq);
2832 
2833 	crq.query_capability.capability = cpu_to_be16(MAX_RX_ADD_QUEUES);
2834 	atomic_inc(&adapter->running_cap_crqs);
2835 	ibmvnic_send_crq(adapter, &crq);
2836 
2837 	crq.query_capability.capability =
2838 	    cpu_to_be16(MIN_TX_ENTRIES_PER_SUBCRQ);
2839 	atomic_inc(&adapter->running_cap_crqs);
2840 	ibmvnic_send_crq(adapter, &crq);
2841 
2842 	crq.query_capability.capability =
2843 	    cpu_to_be16(MIN_RX_ADD_ENTRIES_PER_SUBCRQ);
2844 	atomic_inc(&adapter->running_cap_crqs);
2845 	ibmvnic_send_crq(adapter, &crq);
2846 
2847 	crq.query_capability.capability =
2848 	    cpu_to_be16(MAX_TX_ENTRIES_PER_SUBCRQ);
2849 	atomic_inc(&adapter->running_cap_crqs);
2850 	ibmvnic_send_crq(adapter, &crq);
2851 
2852 	crq.query_capability.capability =
2853 	    cpu_to_be16(MAX_RX_ADD_ENTRIES_PER_SUBCRQ);
2854 	atomic_inc(&adapter->running_cap_crqs);
2855 	ibmvnic_send_crq(adapter, &crq);
2856 
2857 	crq.query_capability.capability = cpu_to_be16(TCP_IP_OFFLOAD);
2858 	atomic_inc(&adapter->running_cap_crqs);
2859 	ibmvnic_send_crq(adapter, &crq);
2860 
2861 	crq.query_capability.capability = cpu_to_be16(PROMISC_SUPPORTED);
2862 	atomic_inc(&adapter->running_cap_crqs);
2863 	ibmvnic_send_crq(adapter, &crq);
2864 
2865 	crq.query_capability.capability = cpu_to_be16(MIN_MTU);
2866 	atomic_inc(&adapter->running_cap_crqs);
2867 	ibmvnic_send_crq(adapter, &crq);
2868 
2869 	crq.query_capability.capability = cpu_to_be16(MAX_MTU);
2870 	atomic_inc(&adapter->running_cap_crqs);
2871 	ibmvnic_send_crq(adapter, &crq);
2872 
2873 	crq.query_capability.capability = cpu_to_be16(MAX_MULTICAST_FILTERS);
2874 	atomic_inc(&adapter->running_cap_crqs);
2875 	ibmvnic_send_crq(adapter, &crq);
2876 
2877 	crq.query_capability.capability = cpu_to_be16(VLAN_HEADER_INSERTION);
2878 	atomic_inc(&adapter->running_cap_crqs);
2879 	ibmvnic_send_crq(adapter, &crq);
2880 
2881 	crq.query_capability.capability = cpu_to_be16(RX_VLAN_HEADER_INSERTION);
2882 	atomic_inc(&adapter->running_cap_crqs);
2883 	ibmvnic_send_crq(adapter, &crq);
2884 
2885 	crq.query_capability.capability = cpu_to_be16(MAX_TX_SG_ENTRIES);
2886 	atomic_inc(&adapter->running_cap_crqs);
2887 	ibmvnic_send_crq(adapter, &crq);
2888 
2889 	crq.query_capability.capability = cpu_to_be16(RX_SG_SUPPORTED);
2890 	atomic_inc(&adapter->running_cap_crqs);
2891 	ibmvnic_send_crq(adapter, &crq);
2892 
2893 	crq.query_capability.capability = cpu_to_be16(OPT_TX_COMP_SUB_QUEUES);
2894 	atomic_inc(&adapter->running_cap_crqs);
2895 	ibmvnic_send_crq(adapter, &crq);
2896 
2897 	crq.query_capability.capability = cpu_to_be16(OPT_RX_COMP_QUEUES);
2898 	atomic_inc(&adapter->running_cap_crqs);
2899 	ibmvnic_send_crq(adapter, &crq);
2900 
2901 	crq.query_capability.capability =
2902 			cpu_to_be16(OPT_RX_BUFADD_Q_PER_RX_COMP_Q);
2903 	atomic_inc(&adapter->running_cap_crqs);
2904 	ibmvnic_send_crq(adapter, &crq);
2905 
2906 	crq.query_capability.capability =
2907 			cpu_to_be16(OPT_TX_ENTRIES_PER_SUBCRQ);
2908 	atomic_inc(&adapter->running_cap_crqs);
2909 	ibmvnic_send_crq(adapter, &crq);
2910 
2911 	crq.query_capability.capability =
2912 			cpu_to_be16(OPT_RXBA_ENTRIES_PER_SUBCRQ);
2913 	atomic_inc(&adapter->running_cap_crqs);
2914 	ibmvnic_send_crq(adapter, &crq);
2915 
2916 	crq.query_capability.capability = cpu_to_be16(TX_RX_DESC_REQ);
2917 	atomic_inc(&adapter->running_cap_crqs);
2918 	ibmvnic_send_crq(adapter, &crq);
2919 }
2920 
handle_query_ip_offload_rsp(struct ibmvnic_adapter * adapter)2921 static void handle_query_ip_offload_rsp(struct ibmvnic_adapter *adapter)
2922 {
2923 	struct device *dev = &adapter->vdev->dev;
2924 	struct ibmvnic_query_ip_offload_buffer *buf = &adapter->ip_offload_buf;
2925 	union ibmvnic_crq crq;
2926 	int i;
2927 
2928 	dma_unmap_single(dev, adapter->ip_offload_tok,
2929 			 sizeof(adapter->ip_offload_buf), DMA_FROM_DEVICE);
2930 
2931 	netdev_dbg(adapter->netdev, "Query IP Offload Buffer:\n");
2932 	for (i = 0; i < (sizeof(adapter->ip_offload_buf) - 1) / 8 + 1; i++)
2933 		netdev_dbg(adapter->netdev, "%016lx\n",
2934 			   ((unsigned long int *)(buf))[i]);
2935 
2936 	netdev_dbg(adapter->netdev, "ipv4_chksum = %d\n", buf->ipv4_chksum);
2937 	netdev_dbg(adapter->netdev, "ipv6_chksum = %d\n", buf->ipv6_chksum);
2938 	netdev_dbg(adapter->netdev, "tcp_ipv4_chksum = %d\n",
2939 		   buf->tcp_ipv4_chksum);
2940 	netdev_dbg(adapter->netdev, "tcp_ipv6_chksum = %d\n",
2941 		   buf->tcp_ipv6_chksum);
2942 	netdev_dbg(adapter->netdev, "udp_ipv4_chksum = %d\n",
2943 		   buf->udp_ipv4_chksum);
2944 	netdev_dbg(adapter->netdev, "udp_ipv6_chksum = %d\n",
2945 		   buf->udp_ipv6_chksum);
2946 	netdev_dbg(adapter->netdev, "large_tx_ipv4 = %d\n",
2947 		   buf->large_tx_ipv4);
2948 	netdev_dbg(adapter->netdev, "large_tx_ipv6 = %d\n",
2949 		   buf->large_tx_ipv6);
2950 	netdev_dbg(adapter->netdev, "large_rx_ipv4 = %d\n",
2951 		   buf->large_rx_ipv4);
2952 	netdev_dbg(adapter->netdev, "large_rx_ipv6 = %d\n",
2953 		   buf->large_rx_ipv6);
2954 	netdev_dbg(adapter->netdev, "max_ipv4_hdr_sz = %d\n",
2955 		   buf->max_ipv4_header_size);
2956 	netdev_dbg(adapter->netdev, "max_ipv6_hdr_sz = %d\n",
2957 		   buf->max_ipv6_header_size);
2958 	netdev_dbg(adapter->netdev, "max_tcp_hdr_size = %d\n",
2959 		   buf->max_tcp_header_size);
2960 	netdev_dbg(adapter->netdev, "max_udp_hdr_size = %d\n",
2961 		   buf->max_udp_header_size);
2962 	netdev_dbg(adapter->netdev, "max_large_tx_size = %d\n",
2963 		   buf->max_large_tx_size);
2964 	netdev_dbg(adapter->netdev, "max_large_rx_size = %d\n",
2965 		   buf->max_large_rx_size);
2966 	netdev_dbg(adapter->netdev, "ipv6_ext_hdr = %d\n",
2967 		   buf->ipv6_extension_header);
2968 	netdev_dbg(adapter->netdev, "tcp_pseudosum_req = %d\n",
2969 		   buf->tcp_pseudosum_req);
2970 	netdev_dbg(adapter->netdev, "num_ipv6_ext_hd = %d\n",
2971 		   buf->num_ipv6_ext_headers);
2972 	netdev_dbg(adapter->netdev, "off_ipv6_ext_hd = %d\n",
2973 		   buf->off_ipv6_ext_headers);
2974 
2975 	adapter->ip_offload_ctrl_tok =
2976 	    dma_map_single(dev, &adapter->ip_offload_ctrl,
2977 			   sizeof(adapter->ip_offload_ctrl), DMA_TO_DEVICE);
2978 
2979 	if (dma_mapping_error(dev, adapter->ip_offload_ctrl_tok)) {
2980 		dev_err(dev, "Couldn't map ip offload control buffer\n");
2981 		return;
2982 	}
2983 
2984 	adapter->ip_offload_ctrl.version = cpu_to_be32(INITIAL_VERSION_IOB);
2985 	adapter->ip_offload_ctrl.tcp_ipv4_chksum = buf->tcp_ipv4_chksum;
2986 	adapter->ip_offload_ctrl.udp_ipv4_chksum = buf->udp_ipv4_chksum;
2987 	adapter->ip_offload_ctrl.tcp_ipv6_chksum = buf->tcp_ipv6_chksum;
2988 	adapter->ip_offload_ctrl.udp_ipv6_chksum = buf->udp_ipv6_chksum;
2989 
2990 	/* large_tx/rx disabled for now, additional features needed */
2991 	adapter->ip_offload_ctrl.large_tx_ipv4 = 0;
2992 	adapter->ip_offload_ctrl.large_tx_ipv6 = 0;
2993 	adapter->ip_offload_ctrl.large_rx_ipv4 = 0;
2994 	adapter->ip_offload_ctrl.large_rx_ipv6 = 0;
2995 
2996 	adapter->netdev->features = NETIF_F_GSO;
2997 
2998 	if (buf->tcp_ipv4_chksum || buf->udp_ipv4_chksum)
2999 		adapter->netdev->features |= NETIF_F_IP_CSUM;
3000 
3001 	if (buf->tcp_ipv6_chksum || buf->udp_ipv6_chksum)
3002 		adapter->netdev->features |= NETIF_F_IPV6_CSUM;
3003 
3004 	if ((adapter->netdev->features &
3005 	    (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)))
3006 		adapter->netdev->features |= NETIF_F_RXCSUM;
3007 
3008 	memset(&crq, 0, sizeof(crq));
3009 	crq.control_ip_offload.first = IBMVNIC_CRQ_CMD;
3010 	crq.control_ip_offload.cmd = CONTROL_IP_OFFLOAD;
3011 	crq.control_ip_offload.len =
3012 	    cpu_to_be32(sizeof(adapter->ip_offload_ctrl));
3013 	crq.control_ip_offload.ioba = cpu_to_be32(adapter->ip_offload_ctrl_tok);
3014 	ibmvnic_send_crq(adapter, &crq);
3015 }
3016 
handle_error_info_rsp(union ibmvnic_crq * crq,struct ibmvnic_adapter * adapter)3017 static void handle_error_info_rsp(union ibmvnic_crq *crq,
3018 				  struct ibmvnic_adapter *adapter)
3019 {
3020 	struct device *dev = &adapter->vdev->dev;
3021 	struct ibmvnic_error_buff *error_buff, *tmp;
3022 	unsigned long flags;
3023 	bool found = false;
3024 	int i;
3025 
3026 	if (!crq->request_error_rsp.rc.code) {
3027 		dev_info(dev, "Request Error Rsp returned with rc=%x\n",
3028 			 crq->request_error_rsp.rc.code);
3029 		return;
3030 	}
3031 
3032 	spin_lock_irqsave(&adapter->error_list_lock, flags);
3033 	list_for_each_entry_safe(error_buff, tmp, &adapter->errors, list)
3034 		if (error_buff->error_id == crq->request_error_rsp.error_id) {
3035 			found = true;
3036 			list_del(&error_buff->list);
3037 			break;
3038 		}
3039 	spin_unlock_irqrestore(&adapter->error_list_lock, flags);
3040 
3041 	if (!found) {
3042 		dev_err(dev, "Couldn't find error id %x\n",
3043 			be32_to_cpu(crq->request_error_rsp.error_id));
3044 		return;
3045 	}
3046 
3047 	dev_err(dev, "Detailed info for error id %x:",
3048 		be32_to_cpu(crq->request_error_rsp.error_id));
3049 
3050 	for (i = 0; i < error_buff->len; i++) {
3051 		pr_cont("%02x", (int)error_buff->buff[i]);
3052 		if (i % 8 == 7)
3053 			pr_cont(" ");
3054 	}
3055 	pr_cont("\n");
3056 
3057 	dma_unmap_single(dev, error_buff->dma, error_buff->len,
3058 			 DMA_FROM_DEVICE);
3059 	kfree(error_buff->buff);
3060 	kfree(error_buff);
3061 }
3062 
request_error_information(struct ibmvnic_adapter * adapter,union ibmvnic_crq * err_crq)3063 static void request_error_information(struct ibmvnic_adapter *adapter,
3064 				      union ibmvnic_crq *err_crq)
3065 {
3066 	struct device *dev = &adapter->vdev->dev;
3067 	struct net_device *netdev = adapter->netdev;
3068 	struct ibmvnic_error_buff *error_buff;
3069 	unsigned long timeout = msecs_to_jiffies(30000);
3070 	union ibmvnic_crq crq;
3071 	unsigned long flags;
3072 	int rc, detail_len;
3073 
3074 	error_buff = kmalloc(sizeof(*error_buff), GFP_ATOMIC);
3075 	if (!error_buff)
3076 		return;
3077 
3078 	detail_len = be32_to_cpu(err_crq->error_indication.detail_error_sz);
3079 	error_buff->buff = kmalloc(detail_len, GFP_ATOMIC);
3080 	if (!error_buff->buff) {
3081 		kfree(error_buff);
3082 		return;
3083 	}
3084 
3085 	error_buff->dma = dma_map_single(dev, error_buff->buff, detail_len,
3086 					 DMA_FROM_DEVICE);
3087 	if (dma_mapping_error(dev, error_buff->dma)) {
3088 		netdev_err(netdev, "Couldn't map error buffer\n");
3089 		kfree(error_buff->buff);
3090 		kfree(error_buff);
3091 		return;
3092 	}
3093 
3094 	error_buff->len = detail_len;
3095 	error_buff->error_id = err_crq->error_indication.error_id;
3096 
3097 	spin_lock_irqsave(&adapter->error_list_lock, flags);
3098 	list_add_tail(&error_buff->list, &adapter->errors);
3099 	spin_unlock_irqrestore(&adapter->error_list_lock, flags);
3100 
3101 	memset(&crq, 0, sizeof(crq));
3102 	crq.request_error_info.first = IBMVNIC_CRQ_CMD;
3103 	crq.request_error_info.cmd = REQUEST_ERROR_INFO;
3104 	crq.request_error_info.ioba = cpu_to_be32(error_buff->dma);
3105 	crq.request_error_info.len = cpu_to_be32(detail_len);
3106 	crq.request_error_info.error_id = err_crq->error_indication.error_id;
3107 
3108 	rc = ibmvnic_send_crq(adapter, &crq);
3109 	if (rc) {
3110 		netdev_err(netdev, "failed to request error information\n");
3111 		goto err_info_fail;
3112 	}
3113 
3114 	if (!wait_for_completion_timeout(&adapter->init_done, timeout)) {
3115 		netdev_err(netdev, "timeout waiting for error information\n");
3116 		goto err_info_fail;
3117 	}
3118 
3119 	return;
3120 
3121 err_info_fail:
3122 	spin_lock_irqsave(&adapter->error_list_lock, flags);
3123 	list_del(&error_buff->list);
3124 	spin_unlock_irqrestore(&adapter->error_list_lock, flags);
3125 
3126 	kfree(error_buff->buff);
3127 	kfree(error_buff);
3128 }
3129 
handle_error_indication(union ibmvnic_crq * crq,struct ibmvnic_adapter * adapter)3130 static void handle_error_indication(union ibmvnic_crq *crq,
3131 				    struct ibmvnic_adapter *adapter)
3132 {
3133 	struct device *dev = &adapter->vdev->dev;
3134 
3135 	dev_err(dev, "Firmware reports %serror id %x, cause %d\n",
3136 		crq->error_indication.flags
3137 			& IBMVNIC_FATAL_ERROR ? "FATAL " : "",
3138 		be32_to_cpu(crq->error_indication.error_id),
3139 		be16_to_cpu(crq->error_indication.error_cause));
3140 
3141 	if (be32_to_cpu(crq->error_indication.error_id))
3142 		request_error_information(adapter, crq);
3143 
3144 	if (crq->error_indication.flags & IBMVNIC_FATAL_ERROR)
3145 		ibmvnic_reset(adapter, VNIC_RESET_FATAL);
3146 	else
3147 		ibmvnic_reset(adapter, VNIC_RESET_NON_FATAL);
3148 }
3149 
handle_change_mac_rsp(union ibmvnic_crq * crq,struct ibmvnic_adapter * adapter)3150 static void handle_change_mac_rsp(union ibmvnic_crq *crq,
3151 				  struct ibmvnic_adapter *adapter)
3152 {
3153 	struct net_device *netdev = adapter->netdev;
3154 	struct device *dev = &adapter->vdev->dev;
3155 	long rc;
3156 
3157 	rc = crq->change_mac_addr_rsp.rc.code;
3158 	if (rc) {
3159 		dev_err(dev, "Error %ld in CHANGE_MAC_ADDR_RSP\n", rc);
3160 		return;
3161 	}
3162 	memcpy(netdev->dev_addr, &crq->change_mac_addr_rsp.mac_addr[0],
3163 	       ETH_ALEN);
3164 }
3165 
handle_request_cap_rsp(union ibmvnic_crq * crq,struct ibmvnic_adapter * adapter)3166 static void handle_request_cap_rsp(union ibmvnic_crq *crq,
3167 				   struct ibmvnic_adapter *adapter)
3168 {
3169 	struct device *dev = &adapter->vdev->dev;
3170 	u64 *req_value;
3171 	char *name;
3172 
3173 	atomic_dec(&adapter->running_cap_crqs);
3174 	switch (be16_to_cpu(crq->request_capability_rsp.capability)) {
3175 	case REQ_TX_QUEUES:
3176 		req_value = &adapter->req_tx_queues;
3177 		name = "tx";
3178 		break;
3179 	case REQ_RX_QUEUES:
3180 		req_value = &adapter->req_rx_queues;
3181 		name = "rx";
3182 		break;
3183 	case REQ_RX_ADD_QUEUES:
3184 		req_value = &adapter->req_rx_add_queues;
3185 		name = "rx_add";
3186 		break;
3187 	case REQ_TX_ENTRIES_PER_SUBCRQ:
3188 		req_value = &adapter->req_tx_entries_per_subcrq;
3189 		name = "tx_entries_per_subcrq";
3190 		break;
3191 	case REQ_RX_ADD_ENTRIES_PER_SUBCRQ:
3192 		req_value = &adapter->req_rx_add_entries_per_subcrq;
3193 		name = "rx_add_entries_per_subcrq";
3194 		break;
3195 	case REQ_MTU:
3196 		req_value = &adapter->req_mtu;
3197 		name = "mtu";
3198 		break;
3199 	case PROMISC_REQUESTED:
3200 		req_value = &adapter->promisc;
3201 		name = "promisc";
3202 		break;
3203 	default:
3204 		dev_err(dev, "Got invalid cap request rsp %d\n",
3205 			crq->request_capability.capability);
3206 		return;
3207 	}
3208 
3209 	switch (crq->request_capability_rsp.rc.code) {
3210 	case SUCCESS:
3211 		break;
3212 	case PARTIALSUCCESS:
3213 		dev_info(dev, "req=%lld, rsp=%ld in %s queue, retrying.\n",
3214 			 *req_value,
3215 			 (long int)be64_to_cpu(crq->request_capability_rsp.
3216 					       number), name);
3217 		*req_value = be64_to_cpu(crq->request_capability_rsp.number);
3218 		ibmvnic_send_req_caps(adapter, 1);
3219 		return;
3220 	default:
3221 		dev_err(dev, "Error %d in request cap rsp\n",
3222 			crq->request_capability_rsp.rc.code);
3223 		return;
3224 	}
3225 
3226 	/* Done receiving requested capabilities, query IP offload support */
3227 	if (atomic_read(&adapter->running_cap_crqs) == 0) {
3228 		union ibmvnic_crq newcrq;
3229 		int buf_sz = sizeof(struct ibmvnic_query_ip_offload_buffer);
3230 		struct ibmvnic_query_ip_offload_buffer *ip_offload_buf =
3231 		    &adapter->ip_offload_buf;
3232 
3233 		adapter->wait_capability = false;
3234 		adapter->ip_offload_tok = dma_map_single(dev, ip_offload_buf,
3235 							 buf_sz,
3236 							 DMA_FROM_DEVICE);
3237 
3238 		if (dma_mapping_error(dev, adapter->ip_offload_tok)) {
3239 			if (!firmware_has_feature(FW_FEATURE_CMO))
3240 				dev_err(dev, "Couldn't map offload buffer\n");
3241 			return;
3242 		}
3243 
3244 		memset(&newcrq, 0, sizeof(newcrq));
3245 		newcrq.query_ip_offload.first = IBMVNIC_CRQ_CMD;
3246 		newcrq.query_ip_offload.cmd = QUERY_IP_OFFLOAD;
3247 		newcrq.query_ip_offload.len = cpu_to_be32(buf_sz);
3248 		newcrq.query_ip_offload.ioba =
3249 		    cpu_to_be32(adapter->ip_offload_tok);
3250 
3251 		ibmvnic_send_crq(adapter, &newcrq);
3252 	}
3253 }
3254 
handle_login_rsp(union ibmvnic_crq * login_rsp_crq,struct ibmvnic_adapter * adapter)3255 static int handle_login_rsp(union ibmvnic_crq *login_rsp_crq,
3256 			    struct ibmvnic_adapter *adapter)
3257 {
3258 	struct device *dev = &adapter->vdev->dev;
3259 	struct ibmvnic_login_rsp_buffer *login_rsp = adapter->login_rsp_buf;
3260 	struct ibmvnic_login_buffer *login = adapter->login_buf;
3261 	int i;
3262 
3263 	dma_unmap_single(dev, adapter->login_buf_token, adapter->login_buf_sz,
3264 			 DMA_BIDIRECTIONAL);
3265 	dma_unmap_single(dev, adapter->login_rsp_buf_token,
3266 			 adapter->login_rsp_buf_sz, DMA_BIDIRECTIONAL);
3267 
3268 	/* If the number of queues requested can't be allocated by the
3269 	 * server, the login response will return with code 1. We will need
3270 	 * to resend the login buffer with fewer queues requested.
3271 	 */
3272 	if (login_rsp_crq->generic.rc.code) {
3273 		adapter->renegotiate = true;
3274 		complete(&adapter->init_done);
3275 		return 0;
3276 	}
3277 
3278 	netdev_dbg(adapter->netdev, "Login Response Buffer:\n");
3279 	for (i = 0; i < (adapter->login_rsp_buf_sz - 1) / 8 + 1; i++) {
3280 		netdev_dbg(adapter->netdev, "%016lx\n",
3281 			   ((unsigned long int *)(adapter->login_rsp_buf))[i]);
3282 	}
3283 
3284 	/* Sanity checks */
3285 	if (login->num_txcomp_subcrqs != login_rsp->num_txsubm_subcrqs ||
3286 	    (be32_to_cpu(login->num_rxcomp_subcrqs) *
3287 	     adapter->req_rx_add_queues !=
3288 	     be32_to_cpu(login_rsp->num_rxadd_subcrqs))) {
3289 		dev_err(dev, "FATAL: Inconsistent login and login rsp\n");
3290 		ibmvnic_remove(adapter->vdev);
3291 		return -EIO;
3292 	}
3293 	complete(&adapter->init_done);
3294 
3295 	return 0;
3296 }
3297 
handle_request_unmap_rsp(union ibmvnic_crq * crq,struct ibmvnic_adapter * adapter)3298 static void handle_request_unmap_rsp(union ibmvnic_crq *crq,
3299 				     struct ibmvnic_adapter *adapter)
3300 {
3301 	struct device *dev = &adapter->vdev->dev;
3302 	long rc;
3303 
3304 	rc = crq->request_unmap_rsp.rc.code;
3305 	if (rc)
3306 		dev_err(dev, "Error %ld in REQUEST_UNMAP_RSP\n", rc);
3307 }
3308 
handle_query_map_rsp(union ibmvnic_crq * crq,struct ibmvnic_adapter * adapter)3309 static void handle_query_map_rsp(union ibmvnic_crq *crq,
3310 				 struct ibmvnic_adapter *adapter)
3311 {
3312 	struct net_device *netdev = adapter->netdev;
3313 	struct device *dev = &adapter->vdev->dev;
3314 	long rc;
3315 
3316 	rc = crq->query_map_rsp.rc.code;
3317 	if (rc) {
3318 		dev_err(dev, "Error %ld in QUERY_MAP_RSP\n", rc);
3319 		return;
3320 	}
3321 	netdev_dbg(netdev, "page_size = %d\ntot_pages = %d\nfree_pages = %d\n",
3322 		   crq->query_map_rsp.page_size, crq->query_map_rsp.tot_pages,
3323 		   crq->query_map_rsp.free_pages);
3324 }
3325 
handle_query_cap_rsp(union ibmvnic_crq * crq,struct ibmvnic_adapter * adapter)3326 static void handle_query_cap_rsp(union ibmvnic_crq *crq,
3327 				 struct ibmvnic_adapter *adapter)
3328 {
3329 	struct net_device *netdev = adapter->netdev;
3330 	struct device *dev = &adapter->vdev->dev;
3331 	long rc;
3332 
3333 	atomic_dec(&adapter->running_cap_crqs);
3334 	netdev_dbg(netdev, "Outstanding queries: %d\n",
3335 		   atomic_read(&adapter->running_cap_crqs));
3336 	rc = crq->query_capability.rc.code;
3337 	if (rc) {
3338 		dev_err(dev, "Error %ld in QUERY_CAP_RSP\n", rc);
3339 		goto out;
3340 	}
3341 
3342 	switch (be16_to_cpu(crq->query_capability.capability)) {
3343 	case MIN_TX_QUEUES:
3344 		adapter->min_tx_queues =
3345 		    be64_to_cpu(crq->query_capability.number);
3346 		netdev_dbg(netdev, "min_tx_queues = %lld\n",
3347 			   adapter->min_tx_queues);
3348 		break;
3349 	case MIN_RX_QUEUES:
3350 		adapter->min_rx_queues =
3351 		    be64_to_cpu(crq->query_capability.number);
3352 		netdev_dbg(netdev, "min_rx_queues = %lld\n",
3353 			   adapter->min_rx_queues);
3354 		break;
3355 	case MIN_RX_ADD_QUEUES:
3356 		adapter->min_rx_add_queues =
3357 		    be64_to_cpu(crq->query_capability.number);
3358 		netdev_dbg(netdev, "min_rx_add_queues = %lld\n",
3359 			   adapter->min_rx_add_queues);
3360 		break;
3361 	case MAX_TX_QUEUES:
3362 		adapter->max_tx_queues =
3363 		    be64_to_cpu(crq->query_capability.number);
3364 		netdev_dbg(netdev, "max_tx_queues = %lld\n",
3365 			   adapter->max_tx_queues);
3366 		break;
3367 	case MAX_RX_QUEUES:
3368 		adapter->max_rx_queues =
3369 		    be64_to_cpu(crq->query_capability.number);
3370 		netdev_dbg(netdev, "max_rx_queues = %lld\n",
3371 			   adapter->max_rx_queues);
3372 		break;
3373 	case MAX_RX_ADD_QUEUES:
3374 		adapter->max_rx_add_queues =
3375 		    be64_to_cpu(crq->query_capability.number);
3376 		netdev_dbg(netdev, "max_rx_add_queues = %lld\n",
3377 			   adapter->max_rx_add_queues);
3378 		break;
3379 	case MIN_TX_ENTRIES_PER_SUBCRQ:
3380 		adapter->min_tx_entries_per_subcrq =
3381 		    be64_to_cpu(crq->query_capability.number);
3382 		netdev_dbg(netdev, "min_tx_entries_per_subcrq = %lld\n",
3383 			   adapter->min_tx_entries_per_subcrq);
3384 		break;
3385 	case MIN_RX_ADD_ENTRIES_PER_SUBCRQ:
3386 		adapter->min_rx_add_entries_per_subcrq =
3387 		    be64_to_cpu(crq->query_capability.number);
3388 		netdev_dbg(netdev, "min_rx_add_entrs_per_subcrq = %lld\n",
3389 			   adapter->min_rx_add_entries_per_subcrq);
3390 		break;
3391 	case MAX_TX_ENTRIES_PER_SUBCRQ:
3392 		adapter->max_tx_entries_per_subcrq =
3393 		    be64_to_cpu(crq->query_capability.number);
3394 		netdev_dbg(netdev, "max_tx_entries_per_subcrq = %lld\n",
3395 			   adapter->max_tx_entries_per_subcrq);
3396 		break;
3397 	case MAX_RX_ADD_ENTRIES_PER_SUBCRQ:
3398 		adapter->max_rx_add_entries_per_subcrq =
3399 		    be64_to_cpu(crq->query_capability.number);
3400 		netdev_dbg(netdev, "max_rx_add_entrs_per_subcrq = %lld\n",
3401 			   adapter->max_rx_add_entries_per_subcrq);
3402 		break;
3403 	case TCP_IP_OFFLOAD:
3404 		adapter->tcp_ip_offload =
3405 		    be64_to_cpu(crq->query_capability.number);
3406 		netdev_dbg(netdev, "tcp_ip_offload = %lld\n",
3407 			   adapter->tcp_ip_offload);
3408 		break;
3409 	case PROMISC_SUPPORTED:
3410 		adapter->promisc_supported =
3411 		    be64_to_cpu(crq->query_capability.number);
3412 		netdev_dbg(netdev, "promisc_supported = %lld\n",
3413 			   adapter->promisc_supported);
3414 		break;
3415 	case MIN_MTU:
3416 		adapter->min_mtu = be64_to_cpu(crq->query_capability.number);
3417 		netdev->min_mtu = adapter->min_mtu - ETH_HLEN;
3418 		netdev_dbg(netdev, "min_mtu = %lld\n", adapter->min_mtu);
3419 		break;
3420 	case MAX_MTU:
3421 		adapter->max_mtu = be64_to_cpu(crq->query_capability.number);
3422 		netdev->max_mtu = adapter->max_mtu - ETH_HLEN;
3423 		netdev_dbg(netdev, "max_mtu = %lld\n", adapter->max_mtu);
3424 		break;
3425 	case MAX_MULTICAST_FILTERS:
3426 		adapter->max_multicast_filters =
3427 		    be64_to_cpu(crq->query_capability.number);
3428 		netdev_dbg(netdev, "max_multicast_filters = %lld\n",
3429 			   adapter->max_multicast_filters);
3430 		break;
3431 	case VLAN_HEADER_INSERTION:
3432 		adapter->vlan_header_insertion =
3433 		    be64_to_cpu(crq->query_capability.number);
3434 		if (adapter->vlan_header_insertion)
3435 			netdev->features |= NETIF_F_HW_VLAN_STAG_TX;
3436 		netdev_dbg(netdev, "vlan_header_insertion = %lld\n",
3437 			   adapter->vlan_header_insertion);
3438 		break;
3439 	case RX_VLAN_HEADER_INSERTION:
3440 		adapter->rx_vlan_header_insertion =
3441 		    be64_to_cpu(crq->query_capability.number);
3442 		netdev_dbg(netdev, "rx_vlan_header_insertion = %lld\n",
3443 			   adapter->rx_vlan_header_insertion);
3444 		break;
3445 	case MAX_TX_SG_ENTRIES:
3446 		adapter->max_tx_sg_entries =
3447 		    be64_to_cpu(crq->query_capability.number);
3448 		netdev_dbg(netdev, "max_tx_sg_entries = %lld\n",
3449 			   adapter->max_tx_sg_entries);
3450 		break;
3451 	case RX_SG_SUPPORTED:
3452 		adapter->rx_sg_supported =
3453 		    be64_to_cpu(crq->query_capability.number);
3454 		netdev_dbg(netdev, "rx_sg_supported = %lld\n",
3455 			   adapter->rx_sg_supported);
3456 		break;
3457 	case OPT_TX_COMP_SUB_QUEUES:
3458 		adapter->opt_tx_comp_sub_queues =
3459 		    be64_to_cpu(crq->query_capability.number);
3460 		netdev_dbg(netdev, "opt_tx_comp_sub_queues = %lld\n",
3461 			   adapter->opt_tx_comp_sub_queues);
3462 		break;
3463 	case OPT_RX_COMP_QUEUES:
3464 		adapter->opt_rx_comp_queues =
3465 		    be64_to_cpu(crq->query_capability.number);
3466 		netdev_dbg(netdev, "opt_rx_comp_queues = %lld\n",
3467 			   adapter->opt_rx_comp_queues);
3468 		break;
3469 	case OPT_RX_BUFADD_Q_PER_RX_COMP_Q:
3470 		adapter->opt_rx_bufadd_q_per_rx_comp_q =
3471 		    be64_to_cpu(crq->query_capability.number);
3472 		netdev_dbg(netdev, "opt_rx_bufadd_q_per_rx_comp_q = %lld\n",
3473 			   adapter->opt_rx_bufadd_q_per_rx_comp_q);
3474 		break;
3475 	case OPT_TX_ENTRIES_PER_SUBCRQ:
3476 		adapter->opt_tx_entries_per_subcrq =
3477 		    be64_to_cpu(crq->query_capability.number);
3478 		netdev_dbg(netdev, "opt_tx_entries_per_subcrq = %lld\n",
3479 			   adapter->opt_tx_entries_per_subcrq);
3480 		break;
3481 	case OPT_RXBA_ENTRIES_PER_SUBCRQ:
3482 		adapter->opt_rxba_entries_per_subcrq =
3483 		    be64_to_cpu(crq->query_capability.number);
3484 		netdev_dbg(netdev, "opt_rxba_entries_per_subcrq = %lld\n",
3485 			   adapter->opt_rxba_entries_per_subcrq);
3486 		break;
3487 	case TX_RX_DESC_REQ:
3488 		adapter->tx_rx_desc_req = crq->query_capability.number;
3489 		netdev_dbg(netdev, "tx_rx_desc_req = %llx\n",
3490 			   adapter->tx_rx_desc_req);
3491 		break;
3492 
3493 	default:
3494 		netdev_err(netdev, "Got invalid cap rsp %d\n",
3495 			   crq->query_capability.capability);
3496 	}
3497 
3498 out:
3499 	if (atomic_read(&adapter->running_cap_crqs) == 0) {
3500 		adapter->wait_capability = false;
3501 		ibmvnic_send_req_caps(adapter, 0);
3502 	}
3503 }
3504 
ibmvnic_handle_crq(union ibmvnic_crq * crq,struct ibmvnic_adapter * adapter)3505 static void ibmvnic_handle_crq(union ibmvnic_crq *crq,
3506 			       struct ibmvnic_adapter *adapter)
3507 {
3508 	struct ibmvnic_generic_crq *gen_crq = &crq->generic;
3509 	struct net_device *netdev = adapter->netdev;
3510 	struct device *dev = &adapter->vdev->dev;
3511 	u64 *u64_crq = (u64 *)crq;
3512 	long rc;
3513 
3514 	netdev_dbg(netdev, "Handling CRQ: %016lx %016lx\n",
3515 		   (unsigned long int)cpu_to_be64(u64_crq[0]),
3516 		   (unsigned long int)cpu_to_be64(u64_crq[1]));
3517 	switch (gen_crq->first) {
3518 	case IBMVNIC_CRQ_INIT_RSP:
3519 		switch (gen_crq->cmd) {
3520 		case IBMVNIC_CRQ_INIT:
3521 			dev_info(dev, "Partner initialized\n");
3522 			adapter->from_passive_init = true;
3523 			complete(&adapter->init_done);
3524 			break;
3525 		case IBMVNIC_CRQ_INIT_COMPLETE:
3526 			dev_info(dev, "Partner initialization complete\n");
3527 			send_version_xchg(adapter);
3528 			break;
3529 		default:
3530 			dev_err(dev, "Unknown crq cmd: %d\n", gen_crq->cmd);
3531 		}
3532 		return;
3533 	case IBMVNIC_CRQ_XPORT_EVENT:
3534 		netif_carrier_off(netdev);
3535 		if (gen_crq->cmd == IBMVNIC_PARTITION_MIGRATED) {
3536 			dev_info(dev, "Migrated, re-enabling adapter\n");
3537 			ibmvnic_reset(adapter, VNIC_RESET_MOBILITY);
3538 		} else if (gen_crq->cmd == IBMVNIC_DEVICE_FAILOVER) {
3539 			dev_info(dev, "Backing device failover detected\n");
3540 			ibmvnic_reset(adapter, VNIC_RESET_FAILOVER);
3541 		} else {
3542 			/* The adapter lost the connection */
3543 			dev_err(dev, "Virtual Adapter failed (rc=%d)\n",
3544 				gen_crq->cmd);
3545 			ibmvnic_reset(adapter, VNIC_RESET_FATAL);
3546 		}
3547 		return;
3548 	case IBMVNIC_CRQ_CMD_RSP:
3549 		break;
3550 	default:
3551 		dev_err(dev, "Got an invalid msg type 0x%02x\n",
3552 			gen_crq->first);
3553 		return;
3554 	}
3555 
3556 	switch (gen_crq->cmd) {
3557 	case VERSION_EXCHANGE_RSP:
3558 		rc = crq->version_exchange_rsp.rc.code;
3559 		if (rc) {
3560 			dev_err(dev, "Error %ld in VERSION_EXCHG_RSP\n", rc);
3561 			break;
3562 		}
3563 		dev_info(dev, "Partner protocol version is %d\n",
3564 			 crq->version_exchange_rsp.version);
3565 		if (be16_to_cpu(crq->version_exchange_rsp.version) <
3566 		    ibmvnic_version)
3567 			ibmvnic_version =
3568 			    be16_to_cpu(crq->version_exchange_rsp.version);
3569 		send_cap_queries(adapter);
3570 		break;
3571 	case QUERY_CAPABILITY_RSP:
3572 		handle_query_cap_rsp(crq, adapter);
3573 		break;
3574 	case QUERY_MAP_RSP:
3575 		handle_query_map_rsp(crq, adapter);
3576 		break;
3577 	case REQUEST_MAP_RSP:
3578 		adapter->fw_done_rc = crq->request_map_rsp.rc.code;
3579 		complete(&adapter->fw_done);
3580 		break;
3581 	case REQUEST_UNMAP_RSP:
3582 		handle_request_unmap_rsp(crq, adapter);
3583 		break;
3584 	case REQUEST_CAPABILITY_RSP:
3585 		handle_request_cap_rsp(crq, adapter);
3586 		break;
3587 	case LOGIN_RSP:
3588 		netdev_dbg(netdev, "Got Login Response\n");
3589 		handle_login_rsp(crq, adapter);
3590 		break;
3591 	case LOGICAL_LINK_STATE_RSP:
3592 		netdev_dbg(netdev,
3593 			   "Got Logical Link State Response, state: %d rc: %d\n",
3594 			   crq->logical_link_state_rsp.link_state,
3595 			   crq->logical_link_state_rsp.rc.code);
3596 		adapter->logical_link_state =
3597 		    crq->logical_link_state_rsp.link_state;
3598 		adapter->init_done_rc = crq->logical_link_state_rsp.rc.code;
3599 		complete(&adapter->init_done);
3600 		break;
3601 	case LINK_STATE_INDICATION:
3602 		netdev_dbg(netdev, "Got Logical Link State Indication\n");
3603 		adapter->phys_link_state =
3604 		    crq->link_state_indication.phys_link_state;
3605 		adapter->logical_link_state =
3606 		    crq->link_state_indication.logical_link_state;
3607 		break;
3608 	case CHANGE_MAC_ADDR_RSP:
3609 		netdev_dbg(netdev, "Got MAC address change Response\n");
3610 		handle_change_mac_rsp(crq, adapter);
3611 		break;
3612 	case ERROR_INDICATION:
3613 		netdev_dbg(netdev, "Got Error Indication\n");
3614 		handle_error_indication(crq, adapter);
3615 		break;
3616 	case REQUEST_ERROR_RSP:
3617 		netdev_dbg(netdev, "Got Error Detail Response\n");
3618 		handle_error_info_rsp(crq, adapter);
3619 		break;
3620 	case REQUEST_STATISTICS_RSP:
3621 		netdev_dbg(netdev, "Got Statistics Response\n");
3622 		complete(&adapter->stats_done);
3623 		break;
3624 	case QUERY_IP_OFFLOAD_RSP:
3625 		netdev_dbg(netdev, "Got Query IP offload Response\n");
3626 		handle_query_ip_offload_rsp(adapter);
3627 		break;
3628 	case MULTICAST_CTRL_RSP:
3629 		netdev_dbg(netdev, "Got multicast control Response\n");
3630 		break;
3631 	case CONTROL_IP_OFFLOAD_RSP:
3632 		netdev_dbg(netdev, "Got Control IP offload Response\n");
3633 		dma_unmap_single(dev, adapter->ip_offload_ctrl_tok,
3634 				 sizeof(adapter->ip_offload_ctrl),
3635 				 DMA_TO_DEVICE);
3636 		complete(&adapter->init_done);
3637 		break;
3638 	case COLLECT_FW_TRACE_RSP:
3639 		netdev_dbg(netdev, "Got Collect firmware trace Response\n");
3640 		complete(&adapter->fw_done);
3641 		break;
3642 	default:
3643 		netdev_err(netdev, "Got an invalid cmd type 0x%02x\n",
3644 			   gen_crq->cmd);
3645 	}
3646 }
3647 
ibmvnic_interrupt(int irq,void * instance)3648 static irqreturn_t ibmvnic_interrupt(int irq, void *instance)
3649 {
3650 	struct ibmvnic_adapter *adapter = instance;
3651 
3652 	tasklet_schedule(&adapter->tasklet);
3653 	return IRQ_HANDLED;
3654 }
3655 
ibmvnic_tasklet(void * data)3656 static void ibmvnic_tasklet(void *data)
3657 {
3658 	struct ibmvnic_adapter *adapter = data;
3659 	struct ibmvnic_crq_queue *queue = &adapter->crq;
3660 	union ibmvnic_crq *crq;
3661 	unsigned long flags;
3662 	bool done = false;
3663 
3664 	spin_lock_irqsave(&queue->lock, flags);
3665 	while (!done) {
3666 		/* Pull all the valid messages off the CRQ */
3667 		while ((crq = ibmvnic_next_crq(adapter)) != NULL) {
3668 			ibmvnic_handle_crq(crq, adapter);
3669 			crq->generic.first = 0;
3670 		}
3671 
3672 		/* remain in tasklet until all
3673 		 * capabilities responses are received
3674 		 */
3675 		if (!adapter->wait_capability)
3676 			done = true;
3677 	}
3678 	/* if capabilities CRQ's were sent in this tasklet, the following
3679 	 * tasklet must wait until all responses are received
3680 	 */
3681 	if (atomic_read(&adapter->running_cap_crqs) != 0)
3682 		adapter->wait_capability = true;
3683 	spin_unlock_irqrestore(&queue->lock, flags);
3684 }
3685 
ibmvnic_reenable_crq_queue(struct ibmvnic_adapter * adapter)3686 static int ibmvnic_reenable_crq_queue(struct ibmvnic_adapter *adapter)
3687 {
3688 	struct vio_dev *vdev = adapter->vdev;
3689 	int rc;
3690 
3691 	do {
3692 		rc = plpar_hcall_norets(H_ENABLE_CRQ, vdev->unit_address);
3693 	} while (rc == H_IN_PROGRESS || rc == H_BUSY || H_IS_LONG_BUSY(rc));
3694 
3695 	if (rc)
3696 		dev_err(&vdev->dev, "Error enabling adapter (rc=%d)\n", rc);
3697 
3698 	return rc;
3699 }
3700 
ibmvnic_reset_crq(struct ibmvnic_adapter * adapter)3701 static int ibmvnic_reset_crq(struct ibmvnic_adapter *adapter)
3702 {
3703 	struct ibmvnic_crq_queue *crq = &adapter->crq;
3704 	struct device *dev = &adapter->vdev->dev;
3705 	struct vio_dev *vdev = adapter->vdev;
3706 	int rc;
3707 
3708 	/* Close the CRQ */
3709 	do {
3710 		rc = plpar_hcall_norets(H_FREE_CRQ, vdev->unit_address);
3711 	} while (rc == H_BUSY || H_IS_LONG_BUSY(rc));
3712 
3713 	/* Clean out the queue */
3714 	memset(crq->msgs, 0, PAGE_SIZE);
3715 	crq->cur = 0;
3716 
3717 	/* And re-open it again */
3718 	rc = plpar_hcall_norets(H_REG_CRQ, vdev->unit_address,
3719 				crq->msg_token, PAGE_SIZE);
3720 
3721 	if (rc == H_CLOSED)
3722 		/* Adapter is good, but other end is not ready */
3723 		dev_warn(dev, "Partner adapter not ready\n");
3724 	else if (rc != 0)
3725 		dev_warn(dev, "Couldn't register crq (rc=%d)\n", rc);
3726 
3727 	return rc;
3728 }
3729 
release_crq_queue(struct ibmvnic_adapter * adapter)3730 static void release_crq_queue(struct ibmvnic_adapter *adapter)
3731 {
3732 	struct ibmvnic_crq_queue *crq = &adapter->crq;
3733 	struct vio_dev *vdev = adapter->vdev;
3734 	long rc;
3735 
3736 	if (!crq->msgs)
3737 		return;
3738 
3739 	netdev_dbg(adapter->netdev, "Releasing CRQ\n");
3740 	free_irq(vdev->irq, adapter);
3741 	tasklet_kill(&adapter->tasklet);
3742 	do {
3743 		rc = plpar_hcall_norets(H_FREE_CRQ, vdev->unit_address);
3744 	} while (rc == H_BUSY || H_IS_LONG_BUSY(rc));
3745 
3746 	dma_unmap_single(&vdev->dev, crq->msg_token, PAGE_SIZE,
3747 			 DMA_BIDIRECTIONAL);
3748 	free_page((unsigned long)crq->msgs);
3749 	crq->msgs = NULL;
3750 }
3751 
init_crq_queue(struct ibmvnic_adapter * adapter)3752 static int init_crq_queue(struct ibmvnic_adapter *adapter)
3753 {
3754 	struct ibmvnic_crq_queue *crq = &adapter->crq;
3755 	struct device *dev = &adapter->vdev->dev;
3756 	struct vio_dev *vdev = adapter->vdev;
3757 	int rc, retrc = -ENOMEM;
3758 
3759 	if (crq->msgs)
3760 		return 0;
3761 
3762 	crq->msgs = (union ibmvnic_crq *)get_zeroed_page(GFP_KERNEL);
3763 	/* Should we allocate more than one page? */
3764 
3765 	if (!crq->msgs)
3766 		return -ENOMEM;
3767 
3768 	crq->size = PAGE_SIZE / sizeof(*crq->msgs);
3769 	crq->msg_token = dma_map_single(dev, crq->msgs, PAGE_SIZE,
3770 					DMA_BIDIRECTIONAL);
3771 	if (dma_mapping_error(dev, crq->msg_token))
3772 		goto map_failed;
3773 
3774 	rc = plpar_hcall_norets(H_REG_CRQ, vdev->unit_address,
3775 				crq->msg_token, PAGE_SIZE);
3776 
3777 	if (rc == H_RESOURCE)
3778 		/* maybe kexecing and resource is busy. try a reset */
3779 		rc = ibmvnic_reset_crq(adapter);
3780 	retrc = rc;
3781 
3782 	if (rc == H_CLOSED) {
3783 		dev_warn(dev, "Partner adapter not ready\n");
3784 	} else if (rc) {
3785 		dev_warn(dev, "Error %d opening adapter\n", rc);
3786 		goto reg_crq_failed;
3787 	}
3788 
3789 	retrc = 0;
3790 
3791 	tasklet_init(&adapter->tasklet, (void *)ibmvnic_tasklet,
3792 		     (unsigned long)adapter);
3793 
3794 	netdev_dbg(adapter->netdev, "registering irq 0x%x\n", vdev->irq);
3795 	rc = request_irq(vdev->irq, ibmvnic_interrupt, 0, IBMVNIC_NAME,
3796 			 adapter);
3797 	if (rc) {
3798 		dev_err(dev, "Couldn't register irq 0x%x. rc=%d\n",
3799 			vdev->irq, rc);
3800 		goto req_irq_failed;
3801 	}
3802 
3803 	rc = vio_enable_interrupts(vdev);
3804 	if (rc) {
3805 		dev_err(dev, "Error %d enabling interrupts\n", rc);
3806 		goto req_irq_failed;
3807 	}
3808 
3809 	crq->cur = 0;
3810 	spin_lock_init(&crq->lock);
3811 
3812 	return retrc;
3813 
3814 req_irq_failed:
3815 	tasklet_kill(&adapter->tasklet);
3816 	do {
3817 		rc = plpar_hcall_norets(H_FREE_CRQ, vdev->unit_address);
3818 	} while (rc == H_BUSY || H_IS_LONG_BUSY(rc));
3819 reg_crq_failed:
3820 	dma_unmap_single(dev, crq->msg_token, PAGE_SIZE, DMA_BIDIRECTIONAL);
3821 map_failed:
3822 	free_page((unsigned long)crq->msgs);
3823 	crq->msgs = NULL;
3824 	return retrc;
3825 }
3826 
ibmvnic_init(struct ibmvnic_adapter * adapter)3827 static int ibmvnic_init(struct ibmvnic_adapter *adapter)
3828 {
3829 	struct device *dev = &adapter->vdev->dev;
3830 	unsigned long timeout = msecs_to_jiffies(30000);
3831 	int rc;
3832 
3833 	if (adapter->resetting) {
3834 		rc = ibmvnic_reset_crq(adapter);
3835 		if (!rc)
3836 			rc = vio_enable_interrupts(adapter->vdev);
3837 	} else {
3838 		rc = init_crq_queue(adapter);
3839 	}
3840 
3841 	if (rc) {
3842 		dev_err(dev, "Couldn't initialize crq. rc=%d\n", rc);
3843 		return rc;
3844 	}
3845 
3846 	adapter->from_passive_init = false;
3847 
3848 	init_completion(&adapter->init_done);
3849 	adapter->init_done_rc = 0;
3850 	ibmvnic_send_crq_init(adapter);
3851 	if (!wait_for_completion_timeout(&adapter->init_done, timeout)) {
3852 		dev_err(dev, "Initialization sequence timed out\n");
3853 		return -1;
3854 	}
3855 
3856 	if (adapter->init_done_rc) {
3857 		release_crq_queue(adapter);
3858 		return adapter->init_done_rc;
3859 	}
3860 
3861 	if (adapter->from_passive_init) {
3862 		adapter->state = VNIC_OPEN;
3863 		adapter->from_passive_init = false;
3864 		return -1;
3865 	}
3866 
3867 	if (adapter->resetting)
3868 		rc = reset_sub_crq_queues(adapter);
3869 	else
3870 		rc = init_sub_crqs(adapter);
3871 	if (rc) {
3872 		dev_err(dev, "Initialization of sub crqs failed\n");
3873 		release_crq_queue(adapter);
3874 		return rc;
3875 	}
3876 
3877 	rc = init_sub_crq_irqs(adapter);
3878 	if (rc) {
3879 		dev_err(dev, "Failed to initialize sub crq irqs\n");
3880 		release_crq_queue(adapter);
3881 	}
3882 
3883 	return rc;
3884 }
3885 
3886 static struct device_attribute dev_attr_failover;
3887 
ibmvnic_probe(struct vio_dev * dev,const struct vio_device_id * id)3888 static int ibmvnic_probe(struct vio_dev *dev, const struct vio_device_id *id)
3889 {
3890 	struct ibmvnic_adapter *adapter;
3891 	struct net_device *netdev;
3892 	unsigned char *mac_addr_p;
3893 	int rc;
3894 
3895 	dev_dbg(&dev->dev, "entering ibmvnic_probe for UA 0x%x\n",
3896 		dev->unit_address);
3897 
3898 	mac_addr_p = (unsigned char *)vio_get_attribute(dev,
3899 							VETH_MAC_ADDR, NULL);
3900 	if (!mac_addr_p) {
3901 		dev_err(&dev->dev,
3902 			"(%s:%3.3d) ERROR: Can't find MAC_ADDR attribute\n",
3903 			__FILE__, __LINE__);
3904 		return 0;
3905 	}
3906 
3907 	netdev = alloc_etherdev_mq(sizeof(struct ibmvnic_adapter),
3908 				   IBMVNIC_MAX_TX_QUEUES);
3909 	if (!netdev)
3910 		return -ENOMEM;
3911 
3912 	adapter = netdev_priv(netdev);
3913 	adapter->state = VNIC_PROBING;
3914 	dev_set_drvdata(&dev->dev, netdev);
3915 	adapter->vdev = dev;
3916 	adapter->netdev = netdev;
3917 
3918 	ether_addr_copy(adapter->mac_addr, mac_addr_p);
3919 	ether_addr_copy(netdev->dev_addr, adapter->mac_addr);
3920 	netdev->irq = dev->irq;
3921 	netdev->netdev_ops = &ibmvnic_netdev_ops;
3922 	netdev->ethtool_ops = &ibmvnic_ethtool_ops;
3923 	SET_NETDEV_DEV(netdev, &dev->dev);
3924 
3925 	spin_lock_init(&adapter->stats_lock);
3926 
3927 	INIT_LIST_HEAD(&adapter->errors);
3928 	spin_lock_init(&adapter->error_list_lock);
3929 
3930 	INIT_WORK(&adapter->ibmvnic_reset, __ibmvnic_reset);
3931 	INIT_LIST_HEAD(&adapter->rwi_list);
3932 	mutex_init(&adapter->reset_lock);
3933 	mutex_init(&adapter->rwi_lock);
3934 	adapter->resetting = false;
3935 
3936 	do {
3937 		rc = ibmvnic_init(adapter);
3938 		if (rc && rc != EAGAIN)
3939 			goto ibmvnic_init_fail;
3940 	} while (rc == EAGAIN);
3941 
3942 	netdev->mtu = adapter->req_mtu - ETH_HLEN;
3943 
3944 	rc = device_create_file(&dev->dev, &dev_attr_failover);
3945 	if (rc)
3946 		goto ibmvnic_init_fail;
3947 
3948 	netif_carrier_off(netdev);
3949 	rc = register_netdev(netdev);
3950 	if (rc) {
3951 		dev_err(&dev->dev, "failed to register netdev rc=%d\n", rc);
3952 		goto ibmvnic_register_fail;
3953 	}
3954 	dev_info(&dev->dev, "ibmvnic registered\n");
3955 
3956 	adapter->state = VNIC_PROBED;
3957 	return 0;
3958 
3959 ibmvnic_register_fail:
3960 	device_remove_file(&dev->dev, &dev_attr_failover);
3961 
3962 ibmvnic_init_fail:
3963 	release_sub_crqs(adapter);
3964 	release_crq_queue(adapter);
3965 	free_netdev(netdev);
3966 
3967 	return rc;
3968 }
3969 
ibmvnic_remove(struct vio_dev * dev)3970 static int ibmvnic_remove(struct vio_dev *dev)
3971 {
3972 	struct net_device *netdev = dev_get_drvdata(&dev->dev);
3973 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
3974 
3975 	adapter->state = VNIC_REMOVING;
3976 	unregister_netdev(netdev);
3977 	mutex_lock(&adapter->reset_lock);
3978 
3979 	release_resources(adapter);
3980 	release_sub_crqs(adapter);
3981 	release_crq_queue(adapter);
3982 
3983 	adapter->state = VNIC_REMOVED;
3984 
3985 	mutex_unlock(&adapter->reset_lock);
3986 	device_remove_file(&dev->dev, &dev_attr_failover);
3987 	free_netdev(netdev);
3988 	dev_set_drvdata(&dev->dev, NULL);
3989 
3990 	return 0;
3991 }
3992 
failover_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)3993 static ssize_t failover_store(struct device *dev, struct device_attribute *attr,
3994 			      const char *buf, size_t count)
3995 {
3996 	struct net_device *netdev = dev_get_drvdata(dev);
3997 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
3998 	unsigned long retbuf[PLPAR_HCALL_BUFSIZE];
3999 	__be64 session_token;
4000 	long rc;
4001 
4002 	if (!sysfs_streq(buf, "1"))
4003 		return -EINVAL;
4004 
4005 	rc = plpar_hcall(H_VIOCTL, retbuf, adapter->vdev->unit_address,
4006 			 H_GET_SESSION_TOKEN, 0, 0, 0);
4007 	if (rc) {
4008 		netdev_err(netdev, "Couldn't retrieve session token, rc %ld\n",
4009 			   rc);
4010 		return -EINVAL;
4011 	}
4012 
4013 	session_token = (__be64)retbuf[0];
4014 	netdev_dbg(netdev, "Initiating client failover, session id %llx\n",
4015 		   be64_to_cpu(session_token));
4016 	rc = plpar_hcall_norets(H_VIOCTL, adapter->vdev->unit_address,
4017 				H_SESSION_ERR_DETECTED, session_token, 0, 0);
4018 	if (rc) {
4019 		netdev_err(netdev, "Client initiated failover failed, rc %ld\n",
4020 			   rc);
4021 		return -EINVAL;
4022 	}
4023 
4024 	return count;
4025 }
4026 
4027 static DEVICE_ATTR(failover, 0200, NULL, failover_store);
4028 
ibmvnic_get_desired_dma(struct vio_dev * vdev)4029 static unsigned long ibmvnic_get_desired_dma(struct vio_dev *vdev)
4030 {
4031 	struct net_device *netdev = dev_get_drvdata(&vdev->dev);
4032 	struct ibmvnic_adapter *adapter;
4033 	struct iommu_table *tbl;
4034 	unsigned long ret = 0;
4035 	int i;
4036 
4037 	tbl = get_iommu_table_base(&vdev->dev);
4038 
4039 	/* netdev inits at probe time along with the structures we need below*/
4040 	if (!netdev)
4041 		return IOMMU_PAGE_ALIGN(IBMVNIC_IO_ENTITLEMENT_DEFAULT, tbl);
4042 
4043 	adapter = netdev_priv(netdev);
4044 
4045 	ret += PAGE_SIZE; /* the crq message queue */
4046 	ret += IOMMU_PAGE_ALIGN(sizeof(struct ibmvnic_statistics), tbl);
4047 
4048 	for (i = 0; i < adapter->req_tx_queues + adapter->req_rx_queues; i++)
4049 		ret += 4 * PAGE_SIZE; /* the scrq message queue */
4050 
4051 	for (i = 0; i < be32_to_cpu(adapter->login_rsp_buf->num_rxadd_subcrqs);
4052 	     i++)
4053 		ret += adapter->rx_pool[i].size *
4054 		    IOMMU_PAGE_ALIGN(adapter->rx_pool[i].buff_size, tbl);
4055 
4056 	return ret;
4057 }
4058 
ibmvnic_resume(struct device * dev)4059 static int ibmvnic_resume(struct device *dev)
4060 {
4061 	struct net_device *netdev = dev_get_drvdata(dev);
4062 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
4063 
4064 	if (adapter->state != VNIC_OPEN)
4065 		return 0;
4066 
4067 	tasklet_schedule(&adapter->tasklet);
4068 
4069 	return 0;
4070 }
4071 
4072 static const struct vio_device_id ibmvnic_device_table[] = {
4073 	{"network", "IBM,vnic"},
4074 	{"", "" }
4075 };
4076 MODULE_DEVICE_TABLE(vio, ibmvnic_device_table);
4077 
4078 static const struct dev_pm_ops ibmvnic_pm_ops = {
4079 	.resume = ibmvnic_resume
4080 };
4081 
4082 static struct vio_driver ibmvnic_driver = {
4083 	.id_table       = ibmvnic_device_table,
4084 	.probe          = ibmvnic_probe,
4085 	.remove         = ibmvnic_remove,
4086 	.get_desired_dma = ibmvnic_get_desired_dma,
4087 	.name		= ibmvnic_driver_name,
4088 	.pm		= &ibmvnic_pm_ops,
4089 };
4090 
4091 /* module functions */
ibmvnic_module_init(void)4092 static int __init ibmvnic_module_init(void)
4093 {
4094 	pr_info("%s: %s %s\n", ibmvnic_driver_name, ibmvnic_driver_string,
4095 		IBMVNIC_DRIVER_VERSION);
4096 
4097 	return vio_register_driver(&ibmvnic_driver);
4098 }
4099 
ibmvnic_module_exit(void)4100 static void __exit ibmvnic_module_exit(void)
4101 {
4102 	vio_unregister_driver(&ibmvnic_driver);
4103 }
4104 
4105 module_init(ibmvnic_module_init);
4106 module_exit(ibmvnic_module_exit);
4107