• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Copyright (c) 2005 Topspin Communications.  All rights reserved.
3  * Copyright (c) 2005, 2006 Cisco Systems.  All rights reserved.
4  * Copyright (c) 2005 Mellanox Technologies. All rights reserved.
5  * Copyright (c) 2005 Voltaire, Inc. All rights reserved.
6  * Copyright (c) 2005 PathScale, Inc. All rights reserved.
7  *
8  * This software is available to you under a choice of one of two
9  * licenses.  You may choose to be licensed under the terms of the GNU
10  * General Public License (GPL) Version 2, available from the file
11  * COPYING in the main directory of this source tree, or the
12  * OpenIB.org BSD license below:
13  *
14  *     Redistribution and use in source and binary forms, with or
15  *     without modification, are permitted provided that the following
16  *     conditions are met:
17  *
18  *      - Redistributions of source code must retain the above
19  *        copyright notice, this list of conditions and the following
20  *        disclaimer.
21  *
22  *      - Redistributions in binary form must reproduce the above
23  *        copyright notice, this list of conditions and the following
24  *        disclaimer in the documentation and/or other materials
25  *        provided with the distribution.
26  *
27  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
28  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
29  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
30  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
31  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
32  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
33  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
34  * SOFTWARE.
35  */
36 
37 #include <linux/module.h>
38 #include <linux/init.h>
39 #include <linux/device.h>
40 #include <linux/err.h>
41 #include <linux/fs.h>
42 #include <linux/poll.h>
43 #include <linux/sched.h>
44 #include <linux/file.h>
45 #include <linux/cdev.h>
46 #include <linux/anon_inodes.h>
47 #include <linux/slab.h>
48 
49 #include <asm/uaccess.h>
50 
51 #include <rdma/ib.h>
52 
53 #include "uverbs.h"
54 
55 MODULE_AUTHOR("Roland Dreier");
56 MODULE_DESCRIPTION("InfiniBand userspace verbs access");
57 MODULE_LICENSE("Dual BSD/GPL");
58 
59 enum {
60 	IB_UVERBS_MAJOR       = 231,
61 	IB_UVERBS_BASE_MINOR  = 192,
62 	IB_UVERBS_MAX_DEVICES = 32
63 };
64 
65 #define IB_UVERBS_BASE_DEV	MKDEV(IB_UVERBS_MAJOR, IB_UVERBS_BASE_MINOR)
66 
67 static struct class *uverbs_class;
68 
69 DEFINE_SPINLOCK(ib_uverbs_idr_lock);
70 DEFINE_IDR(ib_uverbs_pd_idr);
71 DEFINE_IDR(ib_uverbs_mr_idr);
72 DEFINE_IDR(ib_uverbs_mw_idr);
73 DEFINE_IDR(ib_uverbs_ah_idr);
74 DEFINE_IDR(ib_uverbs_cq_idr);
75 DEFINE_IDR(ib_uverbs_qp_idr);
76 DEFINE_IDR(ib_uverbs_srq_idr);
77 DEFINE_IDR(ib_uverbs_xrcd_idr);
78 DEFINE_IDR(ib_uverbs_rule_idr);
79 
80 static DEFINE_SPINLOCK(map_lock);
81 static DECLARE_BITMAP(dev_map, IB_UVERBS_MAX_DEVICES);
82 
83 static ssize_t (*uverbs_cmd_table[])(struct ib_uverbs_file *file,
84 				     struct ib_device *ib_dev,
85 				     const char __user *buf, int in_len,
86 				     int out_len) = {
87 	[IB_USER_VERBS_CMD_GET_CONTEXT]		= ib_uverbs_get_context,
88 	[IB_USER_VERBS_CMD_QUERY_DEVICE]	= ib_uverbs_query_device,
89 	[IB_USER_VERBS_CMD_QUERY_PORT]		= ib_uverbs_query_port,
90 	[IB_USER_VERBS_CMD_ALLOC_PD]		= ib_uverbs_alloc_pd,
91 	[IB_USER_VERBS_CMD_DEALLOC_PD]		= ib_uverbs_dealloc_pd,
92 	[IB_USER_VERBS_CMD_REG_MR]		= ib_uverbs_reg_mr,
93 	[IB_USER_VERBS_CMD_REREG_MR]		= ib_uverbs_rereg_mr,
94 	[IB_USER_VERBS_CMD_DEREG_MR]		= ib_uverbs_dereg_mr,
95 	[IB_USER_VERBS_CMD_ALLOC_MW]		= ib_uverbs_alloc_mw,
96 	[IB_USER_VERBS_CMD_DEALLOC_MW]		= ib_uverbs_dealloc_mw,
97 	[IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL] = ib_uverbs_create_comp_channel,
98 	[IB_USER_VERBS_CMD_CREATE_CQ]		= ib_uverbs_create_cq,
99 	[IB_USER_VERBS_CMD_RESIZE_CQ]		= ib_uverbs_resize_cq,
100 	[IB_USER_VERBS_CMD_POLL_CQ]		= ib_uverbs_poll_cq,
101 	[IB_USER_VERBS_CMD_REQ_NOTIFY_CQ]	= ib_uverbs_req_notify_cq,
102 	[IB_USER_VERBS_CMD_DESTROY_CQ]		= ib_uverbs_destroy_cq,
103 	[IB_USER_VERBS_CMD_CREATE_QP]		= ib_uverbs_create_qp,
104 	[IB_USER_VERBS_CMD_QUERY_QP]		= ib_uverbs_query_qp,
105 	[IB_USER_VERBS_CMD_MODIFY_QP]		= ib_uverbs_modify_qp,
106 	[IB_USER_VERBS_CMD_DESTROY_QP]		= ib_uverbs_destroy_qp,
107 	[IB_USER_VERBS_CMD_POST_SEND]		= ib_uverbs_post_send,
108 	[IB_USER_VERBS_CMD_POST_RECV]		= ib_uverbs_post_recv,
109 	[IB_USER_VERBS_CMD_POST_SRQ_RECV]	= ib_uverbs_post_srq_recv,
110 	[IB_USER_VERBS_CMD_CREATE_AH]		= ib_uverbs_create_ah,
111 	[IB_USER_VERBS_CMD_DESTROY_AH]		= ib_uverbs_destroy_ah,
112 	[IB_USER_VERBS_CMD_ATTACH_MCAST]	= ib_uverbs_attach_mcast,
113 	[IB_USER_VERBS_CMD_DETACH_MCAST]	= ib_uverbs_detach_mcast,
114 	[IB_USER_VERBS_CMD_CREATE_SRQ]		= ib_uverbs_create_srq,
115 	[IB_USER_VERBS_CMD_MODIFY_SRQ]		= ib_uverbs_modify_srq,
116 	[IB_USER_VERBS_CMD_QUERY_SRQ]		= ib_uverbs_query_srq,
117 	[IB_USER_VERBS_CMD_DESTROY_SRQ]		= ib_uverbs_destroy_srq,
118 	[IB_USER_VERBS_CMD_OPEN_XRCD]		= ib_uverbs_open_xrcd,
119 	[IB_USER_VERBS_CMD_CLOSE_XRCD]		= ib_uverbs_close_xrcd,
120 	[IB_USER_VERBS_CMD_CREATE_XSRQ]		= ib_uverbs_create_xsrq,
121 	[IB_USER_VERBS_CMD_OPEN_QP]		= ib_uverbs_open_qp,
122 };
123 
124 static int (*uverbs_ex_cmd_table[])(struct ib_uverbs_file *file,
125 				    struct ib_device *ib_dev,
126 				    struct ib_udata *ucore,
127 				    struct ib_udata *uhw) = {
128 	[IB_USER_VERBS_EX_CMD_CREATE_FLOW]	= ib_uverbs_ex_create_flow,
129 	[IB_USER_VERBS_EX_CMD_DESTROY_FLOW]	= ib_uverbs_ex_destroy_flow,
130 	[IB_USER_VERBS_EX_CMD_QUERY_DEVICE]	= ib_uverbs_ex_query_device,
131 	[IB_USER_VERBS_EX_CMD_CREATE_CQ]	= ib_uverbs_ex_create_cq,
132 	[IB_USER_VERBS_EX_CMD_CREATE_QP]        = ib_uverbs_ex_create_qp,
133 };
134 
135 static void ib_uverbs_add_one(struct ib_device *device);
136 static void ib_uverbs_remove_one(struct ib_device *device, void *client_data);
137 
ib_uverbs_release_dev(struct kobject * kobj)138 static void ib_uverbs_release_dev(struct kobject *kobj)
139 {
140 	struct ib_uverbs_device *dev =
141 		container_of(kobj, struct ib_uverbs_device, kobj);
142 
143 	cleanup_srcu_struct(&dev->disassociate_srcu);
144 	kfree(dev);
145 }
146 
147 static struct kobj_type ib_uverbs_dev_ktype = {
148 	.release = ib_uverbs_release_dev,
149 };
150 
ib_uverbs_release_event_file(struct kref * ref)151 static void ib_uverbs_release_event_file(struct kref *ref)
152 {
153 	struct ib_uverbs_event_file *file =
154 		container_of(ref, struct ib_uverbs_event_file, ref);
155 
156 	kfree(file);
157 }
158 
ib_uverbs_release_ucq(struct ib_uverbs_file * file,struct ib_uverbs_event_file * ev_file,struct ib_ucq_object * uobj)159 void ib_uverbs_release_ucq(struct ib_uverbs_file *file,
160 			  struct ib_uverbs_event_file *ev_file,
161 			  struct ib_ucq_object *uobj)
162 {
163 	struct ib_uverbs_event *evt, *tmp;
164 
165 	if (ev_file) {
166 		spin_lock_irq(&ev_file->lock);
167 		list_for_each_entry_safe(evt, tmp, &uobj->comp_list, obj_list) {
168 			list_del(&evt->list);
169 			kfree(evt);
170 		}
171 		spin_unlock_irq(&ev_file->lock);
172 
173 		kref_put(&ev_file->ref, ib_uverbs_release_event_file);
174 	}
175 
176 	spin_lock_irq(&file->async_file->lock);
177 	list_for_each_entry_safe(evt, tmp, &uobj->async_list, obj_list) {
178 		list_del(&evt->list);
179 		kfree(evt);
180 	}
181 	spin_unlock_irq(&file->async_file->lock);
182 }
183 
ib_uverbs_release_uevent(struct ib_uverbs_file * file,struct ib_uevent_object * uobj)184 void ib_uverbs_release_uevent(struct ib_uverbs_file *file,
185 			      struct ib_uevent_object *uobj)
186 {
187 	struct ib_uverbs_event *evt, *tmp;
188 
189 	spin_lock_irq(&file->async_file->lock);
190 	list_for_each_entry_safe(evt, tmp, &uobj->event_list, obj_list) {
191 		list_del(&evt->list);
192 		kfree(evt);
193 	}
194 	spin_unlock_irq(&file->async_file->lock);
195 }
196 
ib_uverbs_detach_umcast(struct ib_qp * qp,struct ib_uqp_object * uobj)197 static void ib_uverbs_detach_umcast(struct ib_qp *qp,
198 				    struct ib_uqp_object *uobj)
199 {
200 	struct ib_uverbs_mcast_entry *mcast, *tmp;
201 
202 	list_for_each_entry_safe(mcast, tmp, &uobj->mcast_list, list) {
203 		ib_detach_mcast(qp, &mcast->gid, mcast->lid);
204 		list_del(&mcast->list);
205 		kfree(mcast);
206 	}
207 }
208 
ib_uverbs_cleanup_ucontext(struct ib_uverbs_file * file,struct ib_ucontext * context)209 static int ib_uverbs_cleanup_ucontext(struct ib_uverbs_file *file,
210 				      struct ib_ucontext *context)
211 {
212 	struct ib_uobject *uobj, *tmp;
213 
214 	context->closing = 1;
215 
216 	list_for_each_entry_safe(uobj, tmp, &context->ah_list, list) {
217 		struct ib_ah *ah = uobj->object;
218 
219 		idr_remove_uobj(&ib_uverbs_ah_idr, uobj);
220 		ib_destroy_ah(ah);
221 		kfree(uobj);
222 	}
223 
224 	/* Remove MWs before QPs, in order to support type 2A MWs. */
225 	list_for_each_entry_safe(uobj, tmp, &context->mw_list, list) {
226 		struct ib_mw *mw = uobj->object;
227 
228 		idr_remove_uobj(&ib_uverbs_mw_idr, uobj);
229 		ib_dealloc_mw(mw);
230 		kfree(uobj);
231 	}
232 
233 	list_for_each_entry_safe(uobj, tmp, &context->rule_list, list) {
234 		struct ib_flow *flow_id = uobj->object;
235 
236 		idr_remove_uobj(&ib_uverbs_rule_idr, uobj);
237 		ib_destroy_flow(flow_id);
238 		kfree(uobj);
239 	}
240 
241 	list_for_each_entry_safe(uobj, tmp, &context->qp_list, list) {
242 		struct ib_qp *qp = uobj->object;
243 		struct ib_uqp_object *uqp =
244 			container_of(uobj, struct ib_uqp_object, uevent.uobject);
245 
246 		idr_remove_uobj(&ib_uverbs_qp_idr, uobj);
247 		if (qp == qp->real_qp)
248 			ib_uverbs_detach_umcast(qp, uqp);
249 		ib_destroy_qp(qp);
250 		ib_uverbs_release_uevent(file, &uqp->uevent);
251 		kfree(uqp);
252 	}
253 
254 	list_for_each_entry_safe(uobj, tmp, &context->srq_list, list) {
255 		struct ib_srq *srq = uobj->object;
256 		struct ib_uevent_object *uevent =
257 			container_of(uobj, struct ib_uevent_object, uobject);
258 
259 		idr_remove_uobj(&ib_uverbs_srq_idr, uobj);
260 		ib_destroy_srq(srq);
261 		ib_uverbs_release_uevent(file, uevent);
262 		kfree(uevent);
263 	}
264 
265 	list_for_each_entry_safe(uobj, tmp, &context->cq_list, list) {
266 		struct ib_cq *cq = uobj->object;
267 		struct ib_uverbs_event_file *ev_file = cq->cq_context;
268 		struct ib_ucq_object *ucq =
269 			container_of(uobj, struct ib_ucq_object, uobject);
270 
271 		idr_remove_uobj(&ib_uverbs_cq_idr, uobj);
272 		ib_destroy_cq(cq);
273 		ib_uverbs_release_ucq(file, ev_file, ucq);
274 		kfree(ucq);
275 	}
276 
277 	list_for_each_entry_safe(uobj, tmp, &context->mr_list, list) {
278 		struct ib_mr *mr = uobj->object;
279 
280 		idr_remove_uobj(&ib_uverbs_mr_idr, uobj);
281 		ib_dereg_mr(mr);
282 		kfree(uobj);
283 	}
284 
285 	mutex_lock(&file->device->xrcd_tree_mutex);
286 	list_for_each_entry_safe(uobj, tmp, &context->xrcd_list, list) {
287 		struct ib_xrcd *xrcd = uobj->object;
288 		struct ib_uxrcd_object *uxrcd =
289 			container_of(uobj, struct ib_uxrcd_object, uobject);
290 
291 		idr_remove_uobj(&ib_uverbs_xrcd_idr, uobj);
292 		ib_uverbs_dealloc_xrcd(file->device, xrcd);
293 		kfree(uxrcd);
294 	}
295 	mutex_unlock(&file->device->xrcd_tree_mutex);
296 
297 	list_for_each_entry_safe(uobj, tmp, &context->pd_list, list) {
298 		struct ib_pd *pd = uobj->object;
299 
300 		idr_remove_uobj(&ib_uverbs_pd_idr, uobj);
301 		ib_dealloc_pd(pd);
302 		kfree(uobj);
303 	}
304 
305 	put_pid(context->tgid);
306 
307 	return context->device->dealloc_ucontext(context);
308 }
309 
ib_uverbs_comp_dev(struct ib_uverbs_device * dev)310 static void ib_uverbs_comp_dev(struct ib_uverbs_device *dev)
311 {
312 	complete(&dev->comp);
313 }
314 
ib_uverbs_release_file(struct kref * ref)315 static void ib_uverbs_release_file(struct kref *ref)
316 {
317 	struct ib_uverbs_file *file =
318 		container_of(ref, struct ib_uverbs_file, ref);
319 	struct ib_device *ib_dev;
320 	int srcu_key;
321 
322 	srcu_key = srcu_read_lock(&file->device->disassociate_srcu);
323 	ib_dev = srcu_dereference(file->device->ib_dev,
324 				  &file->device->disassociate_srcu);
325 	if (ib_dev && !ib_dev->disassociate_ucontext)
326 		module_put(ib_dev->owner);
327 	srcu_read_unlock(&file->device->disassociate_srcu, srcu_key);
328 
329 	if (atomic_dec_and_test(&file->device->refcount))
330 		ib_uverbs_comp_dev(file->device);
331 
332 	kfree(file);
333 }
334 
ib_uverbs_event_read(struct file * filp,char __user * buf,size_t count,loff_t * pos)335 static ssize_t ib_uverbs_event_read(struct file *filp, char __user *buf,
336 				    size_t count, loff_t *pos)
337 {
338 	struct ib_uverbs_event_file *file = filp->private_data;
339 	struct ib_uverbs_event *event;
340 	int eventsz;
341 	int ret = 0;
342 
343 	spin_lock_irq(&file->lock);
344 
345 	while (list_empty(&file->event_list)) {
346 		spin_unlock_irq(&file->lock);
347 
348 		if (filp->f_flags & O_NONBLOCK)
349 			return -EAGAIN;
350 
351 		if (wait_event_interruptible(file->poll_wait,
352 					     (!list_empty(&file->event_list) ||
353 			/* The barriers built into wait_event_interruptible()
354 			 * and wake_up() guarentee this will see the null set
355 			 * without using RCU
356 			 */
357 					     !file->uverbs_file->device->ib_dev)))
358 			return -ERESTARTSYS;
359 
360 		/* If device was disassociated and no event exists set an error */
361 		if (list_empty(&file->event_list) &&
362 		    !file->uverbs_file->device->ib_dev)
363 			return -EIO;
364 
365 		spin_lock_irq(&file->lock);
366 	}
367 
368 	event = list_entry(file->event_list.next, struct ib_uverbs_event, list);
369 
370 	if (file->is_async)
371 		eventsz = sizeof (struct ib_uverbs_async_event_desc);
372 	else
373 		eventsz = sizeof (struct ib_uverbs_comp_event_desc);
374 
375 	if (eventsz > count) {
376 		ret   = -EINVAL;
377 		event = NULL;
378 	} else {
379 		list_del(file->event_list.next);
380 		if (event->counter) {
381 			++(*event->counter);
382 			list_del(&event->obj_list);
383 		}
384 	}
385 
386 	spin_unlock_irq(&file->lock);
387 
388 	if (event) {
389 		if (copy_to_user(buf, event, eventsz))
390 			ret = -EFAULT;
391 		else
392 			ret = eventsz;
393 	}
394 
395 	kfree(event);
396 
397 	return ret;
398 }
399 
ib_uverbs_event_poll(struct file * filp,struct poll_table_struct * wait)400 static unsigned int ib_uverbs_event_poll(struct file *filp,
401 					 struct poll_table_struct *wait)
402 {
403 	unsigned int pollflags = 0;
404 	struct ib_uverbs_event_file *file = filp->private_data;
405 
406 	poll_wait(filp, &file->poll_wait, wait);
407 
408 	spin_lock_irq(&file->lock);
409 	if (!list_empty(&file->event_list))
410 		pollflags = POLLIN | POLLRDNORM;
411 	spin_unlock_irq(&file->lock);
412 
413 	return pollflags;
414 }
415 
ib_uverbs_event_fasync(int fd,struct file * filp,int on)416 static int ib_uverbs_event_fasync(int fd, struct file *filp, int on)
417 {
418 	struct ib_uverbs_event_file *file = filp->private_data;
419 
420 	return fasync_helper(fd, filp, on, &file->async_queue);
421 }
422 
ib_uverbs_event_close(struct inode * inode,struct file * filp)423 static int ib_uverbs_event_close(struct inode *inode, struct file *filp)
424 {
425 	struct ib_uverbs_event_file *file = filp->private_data;
426 	struct ib_uverbs_event *entry, *tmp;
427 	int closed_already = 0;
428 
429 	mutex_lock(&file->uverbs_file->device->lists_mutex);
430 	spin_lock_irq(&file->lock);
431 	closed_already = file->is_closed;
432 	file->is_closed = 1;
433 	list_for_each_entry_safe(entry, tmp, &file->event_list, list) {
434 		if (entry->counter)
435 			list_del(&entry->obj_list);
436 		kfree(entry);
437 	}
438 	spin_unlock_irq(&file->lock);
439 	if (!closed_already) {
440 		list_del(&file->list);
441 		if (file->is_async)
442 			ib_unregister_event_handler(&file->uverbs_file->
443 				event_handler);
444 	}
445 	mutex_unlock(&file->uverbs_file->device->lists_mutex);
446 
447 	kref_put(&file->uverbs_file->ref, ib_uverbs_release_file);
448 	kref_put(&file->ref, ib_uverbs_release_event_file);
449 
450 	return 0;
451 }
452 
453 static const struct file_operations uverbs_event_fops = {
454 	.owner	 = THIS_MODULE,
455 	.read	 = ib_uverbs_event_read,
456 	.poll    = ib_uverbs_event_poll,
457 	.release = ib_uverbs_event_close,
458 	.fasync  = ib_uverbs_event_fasync,
459 	.llseek	 = no_llseek,
460 };
461 
ib_uverbs_comp_handler(struct ib_cq * cq,void * cq_context)462 void ib_uverbs_comp_handler(struct ib_cq *cq, void *cq_context)
463 {
464 	struct ib_uverbs_event_file    *file = cq_context;
465 	struct ib_ucq_object	       *uobj;
466 	struct ib_uverbs_event	       *entry;
467 	unsigned long			flags;
468 
469 	if (!file)
470 		return;
471 
472 	spin_lock_irqsave(&file->lock, flags);
473 	if (file->is_closed) {
474 		spin_unlock_irqrestore(&file->lock, flags);
475 		return;
476 	}
477 
478 	entry = kmalloc(sizeof *entry, GFP_ATOMIC);
479 	if (!entry) {
480 		spin_unlock_irqrestore(&file->lock, flags);
481 		return;
482 	}
483 
484 	uobj = container_of(cq->uobject, struct ib_ucq_object, uobject);
485 
486 	entry->desc.comp.cq_handle = cq->uobject->user_handle;
487 	entry->counter		   = &uobj->comp_events_reported;
488 
489 	list_add_tail(&entry->list, &file->event_list);
490 	list_add_tail(&entry->obj_list, &uobj->comp_list);
491 	spin_unlock_irqrestore(&file->lock, flags);
492 
493 	wake_up_interruptible(&file->poll_wait);
494 	kill_fasync(&file->async_queue, SIGIO, POLL_IN);
495 }
496 
ib_uverbs_async_handler(struct ib_uverbs_file * file,__u64 element,__u64 event,struct list_head * obj_list,u32 * counter)497 static void ib_uverbs_async_handler(struct ib_uverbs_file *file,
498 				    __u64 element, __u64 event,
499 				    struct list_head *obj_list,
500 				    u32 *counter)
501 {
502 	struct ib_uverbs_event *entry;
503 	unsigned long flags;
504 
505 	spin_lock_irqsave(&file->async_file->lock, flags);
506 	if (file->async_file->is_closed) {
507 		spin_unlock_irqrestore(&file->async_file->lock, flags);
508 		return;
509 	}
510 
511 	entry = kmalloc(sizeof *entry, GFP_ATOMIC);
512 	if (!entry) {
513 		spin_unlock_irqrestore(&file->async_file->lock, flags);
514 		return;
515 	}
516 
517 	entry->desc.async.element    = element;
518 	entry->desc.async.event_type = event;
519 	entry->desc.async.reserved   = 0;
520 	entry->counter               = counter;
521 
522 	list_add_tail(&entry->list, &file->async_file->event_list);
523 	if (obj_list)
524 		list_add_tail(&entry->obj_list, obj_list);
525 	spin_unlock_irqrestore(&file->async_file->lock, flags);
526 
527 	wake_up_interruptible(&file->async_file->poll_wait);
528 	kill_fasync(&file->async_file->async_queue, SIGIO, POLL_IN);
529 }
530 
ib_uverbs_cq_event_handler(struct ib_event * event,void * context_ptr)531 void ib_uverbs_cq_event_handler(struct ib_event *event, void *context_ptr)
532 {
533 	struct ib_ucq_object *uobj = container_of(event->element.cq->uobject,
534 						  struct ib_ucq_object, uobject);
535 
536 	ib_uverbs_async_handler(uobj->uverbs_file, uobj->uobject.user_handle,
537 				event->event, &uobj->async_list,
538 				&uobj->async_events_reported);
539 }
540 
ib_uverbs_qp_event_handler(struct ib_event * event,void * context_ptr)541 void ib_uverbs_qp_event_handler(struct ib_event *event, void *context_ptr)
542 {
543 	struct ib_uevent_object *uobj;
544 
545 	/* for XRC target qp's, check that qp is live */
546 	if (!event->element.qp->uobject || !event->element.qp->uobject->live)
547 		return;
548 
549 	uobj = container_of(event->element.qp->uobject,
550 			    struct ib_uevent_object, uobject);
551 
552 	ib_uverbs_async_handler(context_ptr, uobj->uobject.user_handle,
553 				event->event, &uobj->event_list,
554 				&uobj->events_reported);
555 }
556 
ib_uverbs_srq_event_handler(struct ib_event * event,void * context_ptr)557 void ib_uverbs_srq_event_handler(struct ib_event *event, void *context_ptr)
558 {
559 	struct ib_uevent_object *uobj;
560 
561 	uobj = container_of(event->element.srq->uobject,
562 			    struct ib_uevent_object, uobject);
563 
564 	ib_uverbs_async_handler(context_ptr, uobj->uobject.user_handle,
565 				event->event, &uobj->event_list,
566 				&uobj->events_reported);
567 }
568 
ib_uverbs_event_handler(struct ib_event_handler * handler,struct ib_event * event)569 void ib_uverbs_event_handler(struct ib_event_handler *handler,
570 			     struct ib_event *event)
571 {
572 	struct ib_uverbs_file *file =
573 		container_of(handler, struct ib_uverbs_file, event_handler);
574 
575 	ib_uverbs_async_handler(file, event->element.port_num, event->event,
576 				NULL, NULL);
577 }
578 
ib_uverbs_free_async_event_file(struct ib_uverbs_file * file)579 void ib_uverbs_free_async_event_file(struct ib_uverbs_file *file)
580 {
581 	kref_put(&file->async_file->ref, ib_uverbs_release_event_file);
582 	file->async_file = NULL;
583 }
584 
ib_uverbs_alloc_event_file(struct ib_uverbs_file * uverbs_file,struct ib_device * ib_dev,int is_async)585 struct file *ib_uverbs_alloc_event_file(struct ib_uverbs_file *uverbs_file,
586 					struct ib_device	*ib_dev,
587 					int is_async)
588 {
589 	struct ib_uverbs_event_file *ev_file;
590 	struct file *filp;
591 	int ret;
592 
593 	ev_file = kzalloc(sizeof(*ev_file), GFP_KERNEL);
594 	if (!ev_file)
595 		return ERR_PTR(-ENOMEM);
596 
597 	kref_init(&ev_file->ref);
598 	spin_lock_init(&ev_file->lock);
599 	INIT_LIST_HEAD(&ev_file->event_list);
600 	init_waitqueue_head(&ev_file->poll_wait);
601 	ev_file->uverbs_file = uverbs_file;
602 	kref_get(&ev_file->uverbs_file->ref);
603 	ev_file->async_queue = NULL;
604 	ev_file->is_closed   = 0;
605 
606 	filp = anon_inode_getfile("[infinibandevent]", &uverbs_event_fops,
607 				  ev_file, O_RDONLY);
608 	if (IS_ERR(filp))
609 		goto err_put_refs;
610 
611 	mutex_lock(&uverbs_file->device->lists_mutex);
612 	list_add_tail(&ev_file->list,
613 		      &uverbs_file->device->uverbs_events_file_list);
614 	mutex_unlock(&uverbs_file->device->lists_mutex);
615 
616 	if (is_async) {
617 		WARN_ON(uverbs_file->async_file);
618 		uverbs_file->async_file = ev_file;
619 		kref_get(&uverbs_file->async_file->ref);
620 		INIT_IB_EVENT_HANDLER(&uverbs_file->event_handler,
621 				      ib_dev,
622 				      ib_uverbs_event_handler);
623 		ret = ib_register_event_handler(&uverbs_file->event_handler);
624 		if (ret)
625 			goto err_put_file;
626 
627 		/* At that point async file stuff was fully set */
628 		ev_file->is_async = 1;
629 	}
630 
631 	return filp;
632 
633 err_put_file:
634 	fput(filp);
635 	kref_put(&uverbs_file->async_file->ref, ib_uverbs_release_event_file);
636 	uverbs_file->async_file = NULL;
637 	return ERR_PTR(ret);
638 
639 err_put_refs:
640 	kref_put(&ev_file->uverbs_file->ref, ib_uverbs_release_file);
641 	kref_put(&ev_file->ref, ib_uverbs_release_event_file);
642 	return filp;
643 }
644 
645 /*
646  * Look up a completion event file by FD.  If lookup is successful,
647  * takes a ref to the event file struct that it returns; if
648  * unsuccessful, returns NULL.
649  */
ib_uverbs_lookup_comp_file(int fd)650 struct ib_uverbs_event_file *ib_uverbs_lookup_comp_file(int fd)
651 {
652 	struct ib_uverbs_event_file *ev_file = NULL;
653 	struct fd f = fdget(fd);
654 
655 	if (!f.file)
656 		return NULL;
657 
658 	if (f.file->f_op != &uverbs_event_fops)
659 		goto out;
660 
661 	ev_file = f.file->private_data;
662 	if (ev_file->is_async) {
663 		ev_file = NULL;
664 		goto out;
665 	}
666 
667 	kref_get(&ev_file->ref);
668 
669 out:
670 	fdput(f);
671 	return ev_file;
672 }
673 
ib_uverbs_write(struct file * filp,const char __user * buf,size_t count,loff_t * pos)674 static ssize_t ib_uverbs_write(struct file *filp, const char __user *buf,
675 			     size_t count, loff_t *pos)
676 {
677 	struct ib_uverbs_file *file = filp->private_data;
678 	struct ib_device *ib_dev;
679 	struct ib_uverbs_cmd_hdr hdr;
680 	__u32 flags;
681 	int srcu_key;
682 	ssize_t ret;
683 
684 	if (WARN_ON_ONCE(!ib_safe_file_access(filp)))
685 		return -EACCES;
686 
687 	if (count < sizeof hdr)
688 		return -EINVAL;
689 
690 	if (copy_from_user(&hdr, buf, sizeof hdr))
691 		return -EFAULT;
692 
693 	srcu_key = srcu_read_lock(&file->device->disassociate_srcu);
694 	ib_dev = srcu_dereference(file->device->ib_dev,
695 				  &file->device->disassociate_srcu);
696 	if (!ib_dev) {
697 		ret = -EIO;
698 		goto out;
699 	}
700 
701 	flags = (hdr.command &
702 		 IB_USER_VERBS_CMD_FLAGS_MASK) >> IB_USER_VERBS_CMD_FLAGS_SHIFT;
703 
704 	if (!flags) {
705 		__u32 command;
706 
707 		if (hdr.command & ~(__u32)(IB_USER_VERBS_CMD_FLAGS_MASK |
708 					   IB_USER_VERBS_CMD_COMMAND_MASK)) {
709 			ret = -EINVAL;
710 			goto out;
711 		}
712 
713 		command = hdr.command & IB_USER_VERBS_CMD_COMMAND_MASK;
714 
715 		if (command >= ARRAY_SIZE(uverbs_cmd_table) ||
716 		    !uverbs_cmd_table[command]) {
717 			ret = -EINVAL;
718 			goto out;
719 		}
720 
721 		if (!file->ucontext &&
722 		    command != IB_USER_VERBS_CMD_GET_CONTEXT) {
723 			ret = -EINVAL;
724 			goto out;
725 		}
726 
727 		if (!(ib_dev->uverbs_cmd_mask & (1ull << command))) {
728 			ret = -ENOSYS;
729 			goto out;
730 		}
731 
732 		if (hdr.in_words * 4 != count) {
733 			ret = -EINVAL;
734 			goto out;
735 		}
736 
737 		ret = uverbs_cmd_table[command](file, ib_dev,
738 						 buf + sizeof(hdr),
739 						 hdr.in_words * 4,
740 						 hdr.out_words * 4);
741 
742 	} else if (flags == IB_USER_VERBS_CMD_FLAG_EXTENDED) {
743 		__u32 command;
744 
745 		struct ib_uverbs_ex_cmd_hdr ex_hdr;
746 		struct ib_udata ucore;
747 		struct ib_udata uhw;
748 		size_t written_count = count;
749 
750 		if (hdr.command & ~(__u32)(IB_USER_VERBS_CMD_FLAGS_MASK |
751 					   IB_USER_VERBS_CMD_COMMAND_MASK)) {
752 			ret = -EINVAL;
753 			goto out;
754 		}
755 
756 		command = hdr.command & IB_USER_VERBS_CMD_COMMAND_MASK;
757 
758 		if (command >= ARRAY_SIZE(uverbs_ex_cmd_table) ||
759 		    !uverbs_ex_cmd_table[command]) {
760 			ret = -ENOSYS;
761 			goto out;
762 		}
763 
764 		if (!file->ucontext) {
765 			ret = -EINVAL;
766 			goto out;
767 		}
768 
769 		if (!(ib_dev->uverbs_ex_cmd_mask & (1ull << command))) {
770 			ret = -ENOSYS;
771 			goto out;
772 		}
773 
774 		if (count < (sizeof(hdr) + sizeof(ex_hdr))) {
775 			ret = -EINVAL;
776 			goto out;
777 		}
778 
779 		if (copy_from_user(&ex_hdr, buf + sizeof(hdr), sizeof(ex_hdr))) {
780 			ret = -EFAULT;
781 			goto out;
782 		}
783 
784 		count -= sizeof(hdr) + sizeof(ex_hdr);
785 		buf += sizeof(hdr) + sizeof(ex_hdr);
786 
787 		if ((hdr.in_words + ex_hdr.provider_in_words) * 8 != count) {
788 			ret = -EINVAL;
789 			goto out;
790 		}
791 
792 		if (ex_hdr.cmd_hdr_reserved) {
793 			ret = -EINVAL;
794 			goto out;
795 		}
796 
797 		if (ex_hdr.response) {
798 			if (!hdr.out_words && !ex_hdr.provider_out_words) {
799 				ret = -EINVAL;
800 				goto out;
801 			}
802 
803 			if (!access_ok(VERIFY_WRITE,
804 				       (void __user *) (unsigned long) ex_hdr.response,
805 				       (hdr.out_words + ex_hdr.provider_out_words) * 8)) {
806 				ret = -EFAULT;
807 				goto out;
808 			}
809 		} else {
810 			if (hdr.out_words || ex_hdr.provider_out_words) {
811 				ret = -EINVAL;
812 				goto out;
813 			}
814 		}
815 
816 		INIT_UDATA_BUF_OR_NULL(&ucore, buf, (unsigned long) ex_hdr.response,
817 				       hdr.in_words * 8, hdr.out_words * 8);
818 
819 		INIT_UDATA_BUF_OR_NULL(&uhw,
820 				       buf + ucore.inlen,
821 				       (unsigned long) ex_hdr.response + ucore.outlen,
822 				       ex_hdr.provider_in_words * 8,
823 				       ex_hdr.provider_out_words * 8);
824 
825 		ret = uverbs_ex_cmd_table[command](file,
826 						   ib_dev,
827 						   &ucore,
828 						   &uhw);
829 		if (!ret)
830 			ret = written_count;
831 	} else {
832 		ret = -ENOSYS;
833 	}
834 
835 out:
836 	srcu_read_unlock(&file->device->disassociate_srcu, srcu_key);
837 	return ret;
838 }
839 
ib_uverbs_mmap(struct file * filp,struct vm_area_struct * vma)840 static int ib_uverbs_mmap(struct file *filp, struct vm_area_struct *vma)
841 {
842 	struct ib_uverbs_file *file = filp->private_data;
843 	struct ib_device *ib_dev;
844 	int ret = 0;
845 	int srcu_key;
846 
847 	srcu_key = srcu_read_lock(&file->device->disassociate_srcu);
848 	ib_dev = srcu_dereference(file->device->ib_dev,
849 				  &file->device->disassociate_srcu);
850 	if (!ib_dev) {
851 		ret = -EIO;
852 		goto out;
853 	}
854 
855 	if (!file->ucontext)
856 		ret = -ENODEV;
857 	else
858 		ret = ib_dev->mmap(file->ucontext, vma);
859 out:
860 	srcu_read_unlock(&file->device->disassociate_srcu, srcu_key);
861 	return ret;
862 }
863 
864 /*
865  * ib_uverbs_open() does not need the BKL:
866  *
867  *  - the ib_uverbs_device structures are properly reference counted and
868  *    everything else is purely local to the file being created, so
869  *    races against other open calls are not a problem;
870  *  - there is no ioctl method to race against;
871  *  - the open method will either immediately run -ENXIO, or all
872  *    required initialization will be done.
873  */
ib_uverbs_open(struct inode * inode,struct file * filp)874 static int ib_uverbs_open(struct inode *inode, struct file *filp)
875 {
876 	struct ib_uverbs_device *dev;
877 	struct ib_uverbs_file *file;
878 	struct ib_device *ib_dev;
879 	int ret;
880 	int module_dependent;
881 	int srcu_key;
882 
883 	dev = container_of(inode->i_cdev, struct ib_uverbs_device, cdev);
884 	if (!atomic_inc_not_zero(&dev->refcount))
885 		return -ENXIO;
886 
887 	srcu_key = srcu_read_lock(&dev->disassociate_srcu);
888 	mutex_lock(&dev->lists_mutex);
889 	ib_dev = srcu_dereference(dev->ib_dev,
890 				  &dev->disassociate_srcu);
891 	if (!ib_dev) {
892 		ret = -EIO;
893 		goto err;
894 	}
895 
896 	/* In case IB device supports disassociate ucontext, there is no hard
897 	 * dependency between uverbs device and its low level device.
898 	 */
899 	module_dependent = !(ib_dev->disassociate_ucontext);
900 
901 	if (module_dependent) {
902 		if (!try_module_get(ib_dev->owner)) {
903 			ret = -ENODEV;
904 			goto err;
905 		}
906 	}
907 
908 	file = kzalloc(sizeof(*file), GFP_KERNEL);
909 	if (!file) {
910 		ret = -ENOMEM;
911 		if (module_dependent)
912 			goto err_module;
913 
914 		goto err;
915 	}
916 
917 	file->device	 = dev;
918 	file->ucontext	 = NULL;
919 	file->async_file = NULL;
920 	kref_init(&file->ref);
921 	mutex_init(&file->mutex);
922 	mutex_init(&file->cleanup_mutex);
923 
924 	filp->private_data = file;
925 	kobject_get(&dev->kobj);
926 	list_add_tail(&file->list, &dev->uverbs_file_list);
927 	mutex_unlock(&dev->lists_mutex);
928 	srcu_read_unlock(&dev->disassociate_srcu, srcu_key);
929 
930 	return nonseekable_open(inode, filp);
931 
932 err_module:
933 	module_put(ib_dev->owner);
934 
935 err:
936 	mutex_unlock(&dev->lists_mutex);
937 	srcu_read_unlock(&dev->disassociate_srcu, srcu_key);
938 	if (atomic_dec_and_test(&dev->refcount))
939 		ib_uverbs_comp_dev(dev);
940 
941 	return ret;
942 }
943 
ib_uverbs_close(struct inode * inode,struct file * filp)944 static int ib_uverbs_close(struct inode *inode, struct file *filp)
945 {
946 	struct ib_uverbs_file *file = filp->private_data;
947 	struct ib_uverbs_device *dev = file->device;
948 
949 	mutex_lock(&file->cleanup_mutex);
950 	if (file->ucontext) {
951 		ib_uverbs_cleanup_ucontext(file, file->ucontext);
952 		file->ucontext = NULL;
953 	}
954 	mutex_unlock(&file->cleanup_mutex);
955 
956 	mutex_lock(&file->device->lists_mutex);
957 	if (!file->is_closed) {
958 		list_del(&file->list);
959 		file->is_closed = 1;
960 	}
961 	mutex_unlock(&file->device->lists_mutex);
962 
963 	if (file->async_file)
964 		kref_put(&file->async_file->ref, ib_uverbs_release_event_file);
965 
966 	kref_put(&file->ref, ib_uverbs_release_file);
967 	kobject_put(&dev->kobj);
968 
969 	return 0;
970 }
971 
972 static const struct file_operations uverbs_fops = {
973 	.owner	 = THIS_MODULE,
974 	.write	 = ib_uverbs_write,
975 	.open	 = ib_uverbs_open,
976 	.release = ib_uverbs_close,
977 	.llseek	 = no_llseek,
978 };
979 
980 static const struct file_operations uverbs_mmap_fops = {
981 	.owner	 = THIS_MODULE,
982 	.write	 = ib_uverbs_write,
983 	.mmap    = ib_uverbs_mmap,
984 	.open	 = ib_uverbs_open,
985 	.release = ib_uverbs_close,
986 	.llseek	 = no_llseek,
987 };
988 
989 static struct ib_client uverbs_client = {
990 	.name   = "uverbs",
991 	.add    = ib_uverbs_add_one,
992 	.remove = ib_uverbs_remove_one
993 };
994 
show_ibdev(struct device * device,struct device_attribute * attr,char * buf)995 static ssize_t show_ibdev(struct device *device, struct device_attribute *attr,
996 			  char *buf)
997 {
998 	int ret = -ENODEV;
999 	int srcu_key;
1000 	struct ib_uverbs_device *dev = dev_get_drvdata(device);
1001 	struct ib_device *ib_dev;
1002 
1003 	if (!dev)
1004 		return -ENODEV;
1005 
1006 	srcu_key = srcu_read_lock(&dev->disassociate_srcu);
1007 	ib_dev = srcu_dereference(dev->ib_dev, &dev->disassociate_srcu);
1008 	if (ib_dev)
1009 		ret = sprintf(buf, "%s\n", ib_dev->name);
1010 	srcu_read_unlock(&dev->disassociate_srcu, srcu_key);
1011 
1012 	return ret;
1013 }
1014 static DEVICE_ATTR(ibdev, S_IRUGO, show_ibdev, NULL);
1015 
show_dev_abi_version(struct device * device,struct device_attribute * attr,char * buf)1016 static ssize_t show_dev_abi_version(struct device *device,
1017 				    struct device_attribute *attr, char *buf)
1018 {
1019 	struct ib_uverbs_device *dev = dev_get_drvdata(device);
1020 	int ret = -ENODEV;
1021 	int srcu_key;
1022 	struct ib_device *ib_dev;
1023 
1024 	if (!dev)
1025 		return -ENODEV;
1026 	srcu_key = srcu_read_lock(&dev->disassociate_srcu);
1027 	ib_dev = srcu_dereference(dev->ib_dev, &dev->disassociate_srcu);
1028 	if (ib_dev)
1029 		ret = sprintf(buf, "%d\n", ib_dev->uverbs_abi_ver);
1030 	srcu_read_unlock(&dev->disassociate_srcu, srcu_key);
1031 
1032 	return ret;
1033 }
1034 static DEVICE_ATTR(abi_version, S_IRUGO, show_dev_abi_version, NULL);
1035 
1036 static CLASS_ATTR_STRING(abi_version, S_IRUGO,
1037 			 __stringify(IB_USER_VERBS_ABI_VERSION));
1038 
1039 static dev_t overflow_maj;
1040 static DECLARE_BITMAP(overflow_map, IB_UVERBS_MAX_DEVICES);
1041 
1042 /*
1043  * If we have more than IB_UVERBS_MAX_DEVICES, dynamically overflow by
1044  * requesting a new major number and doubling the number of max devices we
1045  * support. It's stupid, but simple.
1046  */
find_overflow_devnum(void)1047 static int find_overflow_devnum(void)
1048 {
1049 	int ret;
1050 
1051 	if (!overflow_maj) {
1052 		ret = alloc_chrdev_region(&overflow_maj, 0, IB_UVERBS_MAX_DEVICES,
1053 					  "infiniband_verbs");
1054 		if (ret) {
1055 			printk(KERN_ERR "user_verbs: couldn't register dynamic device number\n");
1056 			return ret;
1057 		}
1058 	}
1059 
1060 	ret = find_first_zero_bit(overflow_map, IB_UVERBS_MAX_DEVICES);
1061 	if (ret >= IB_UVERBS_MAX_DEVICES)
1062 		return -1;
1063 
1064 	return ret;
1065 }
1066 
ib_uverbs_add_one(struct ib_device * device)1067 static void ib_uverbs_add_one(struct ib_device *device)
1068 {
1069 	int devnum;
1070 	dev_t base;
1071 	struct ib_uverbs_device *uverbs_dev;
1072 	int ret;
1073 
1074 	if (!device->alloc_ucontext)
1075 		return;
1076 
1077 	uverbs_dev = kzalloc(sizeof *uverbs_dev, GFP_KERNEL);
1078 	if (!uverbs_dev)
1079 		return;
1080 
1081 	ret = init_srcu_struct(&uverbs_dev->disassociate_srcu);
1082 	if (ret) {
1083 		kfree(uverbs_dev);
1084 		return;
1085 	}
1086 
1087 	atomic_set(&uverbs_dev->refcount, 1);
1088 	init_completion(&uverbs_dev->comp);
1089 	uverbs_dev->xrcd_tree = RB_ROOT;
1090 	mutex_init(&uverbs_dev->xrcd_tree_mutex);
1091 	kobject_init(&uverbs_dev->kobj, &ib_uverbs_dev_ktype);
1092 	mutex_init(&uverbs_dev->lists_mutex);
1093 	INIT_LIST_HEAD(&uverbs_dev->uverbs_file_list);
1094 	INIT_LIST_HEAD(&uverbs_dev->uverbs_events_file_list);
1095 
1096 	spin_lock(&map_lock);
1097 	devnum = find_first_zero_bit(dev_map, IB_UVERBS_MAX_DEVICES);
1098 	if (devnum >= IB_UVERBS_MAX_DEVICES) {
1099 		spin_unlock(&map_lock);
1100 		devnum = find_overflow_devnum();
1101 		if (devnum < 0)
1102 			goto err;
1103 
1104 		spin_lock(&map_lock);
1105 		uverbs_dev->devnum = devnum + IB_UVERBS_MAX_DEVICES;
1106 		base = devnum + overflow_maj;
1107 		set_bit(devnum, overflow_map);
1108 	} else {
1109 		uverbs_dev->devnum = devnum;
1110 		base = devnum + IB_UVERBS_BASE_DEV;
1111 		set_bit(devnum, dev_map);
1112 	}
1113 	spin_unlock(&map_lock);
1114 
1115 	rcu_assign_pointer(uverbs_dev->ib_dev, device);
1116 	uverbs_dev->num_comp_vectors = device->num_comp_vectors;
1117 
1118 	cdev_init(&uverbs_dev->cdev, NULL);
1119 	uverbs_dev->cdev.owner = THIS_MODULE;
1120 	uverbs_dev->cdev.ops = device->mmap ? &uverbs_mmap_fops : &uverbs_fops;
1121 	uverbs_dev->cdev.kobj.parent = &uverbs_dev->kobj;
1122 	kobject_set_name(&uverbs_dev->cdev.kobj, "uverbs%d", uverbs_dev->devnum);
1123 	if (cdev_add(&uverbs_dev->cdev, base, 1))
1124 		goto err_cdev;
1125 
1126 	uverbs_dev->dev = device_create(uverbs_class, device->dma_device,
1127 					uverbs_dev->cdev.dev, uverbs_dev,
1128 					"uverbs%d", uverbs_dev->devnum);
1129 	if (IS_ERR(uverbs_dev->dev))
1130 		goto err_cdev;
1131 
1132 	if (device_create_file(uverbs_dev->dev, &dev_attr_ibdev))
1133 		goto err_class;
1134 	if (device_create_file(uverbs_dev->dev, &dev_attr_abi_version))
1135 		goto err_class;
1136 
1137 	ib_set_client_data(device, &uverbs_client, uverbs_dev);
1138 
1139 	return;
1140 
1141 err_class:
1142 	device_destroy(uverbs_class, uverbs_dev->cdev.dev);
1143 
1144 err_cdev:
1145 	cdev_del(&uverbs_dev->cdev);
1146 	if (uverbs_dev->devnum < IB_UVERBS_MAX_DEVICES)
1147 		clear_bit(devnum, dev_map);
1148 	else
1149 		clear_bit(devnum, overflow_map);
1150 
1151 err:
1152 	if (atomic_dec_and_test(&uverbs_dev->refcount))
1153 		ib_uverbs_comp_dev(uverbs_dev);
1154 	wait_for_completion(&uverbs_dev->comp);
1155 	kobject_put(&uverbs_dev->kobj);
1156 	return;
1157 }
1158 
ib_uverbs_free_hw_resources(struct ib_uverbs_device * uverbs_dev,struct ib_device * ib_dev)1159 static void ib_uverbs_free_hw_resources(struct ib_uverbs_device *uverbs_dev,
1160 					struct ib_device *ib_dev)
1161 {
1162 	struct ib_uverbs_file *file;
1163 	struct ib_uverbs_event_file *event_file;
1164 	struct ib_event event;
1165 
1166 	/* Pending running commands to terminate */
1167 	synchronize_srcu(&uverbs_dev->disassociate_srcu);
1168 	event.event = IB_EVENT_DEVICE_FATAL;
1169 	event.element.port_num = 0;
1170 	event.device = ib_dev;
1171 
1172 	mutex_lock(&uverbs_dev->lists_mutex);
1173 	while (!list_empty(&uverbs_dev->uverbs_file_list)) {
1174 		struct ib_ucontext *ucontext;
1175 		file = list_first_entry(&uverbs_dev->uverbs_file_list,
1176 					struct ib_uverbs_file, list);
1177 		file->is_closed = 1;
1178 		list_del(&file->list);
1179 		kref_get(&file->ref);
1180 		mutex_unlock(&uverbs_dev->lists_mutex);
1181 
1182 		ib_uverbs_event_handler(&file->event_handler, &event);
1183 
1184 		mutex_lock(&file->cleanup_mutex);
1185 		ucontext = file->ucontext;
1186 		file->ucontext = NULL;
1187 		mutex_unlock(&file->cleanup_mutex);
1188 
1189 		/* At this point ib_uverbs_close cannot be running
1190 		 * ib_uverbs_cleanup_ucontext
1191 		 */
1192 		if (ucontext) {
1193 			/* We must release the mutex before going ahead and
1194 			 * calling disassociate_ucontext. disassociate_ucontext
1195 			 * might end up indirectly calling uverbs_close,
1196 			 * for example due to freeing the resources
1197 			 * (e.g mmput).
1198 			 */
1199 			ib_dev->disassociate_ucontext(ucontext);
1200 			ib_uverbs_cleanup_ucontext(file, ucontext);
1201 		}
1202 
1203 		mutex_lock(&uverbs_dev->lists_mutex);
1204 		kref_put(&file->ref, ib_uverbs_release_file);
1205 	}
1206 
1207 	while (!list_empty(&uverbs_dev->uverbs_events_file_list)) {
1208 		event_file = list_first_entry(&uverbs_dev->
1209 					      uverbs_events_file_list,
1210 					      struct ib_uverbs_event_file,
1211 					      list);
1212 		spin_lock_irq(&event_file->lock);
1213 		event_file->is_closed = 1;
1214 		spin_unlock_irq(&event_file->lock);
1215 
1216 		list_del(&event_file->list);
1217 		if (event_file->is_async) {
1218 			ib_unregister_event_handler(&event_file->uverbs_file->
1219 						    event_handler);
1220 			event_file->uverbs_file->event_handler.device = NULL;
1221 		}
1222 
1223 		wake_up_interruptible(&event_file->poll_wait);
1224 		kill_fasync(&event_file->async_queue, SIGIO, POLL_IN);
1225 	}
1226 	mutex_unlock(&uverbs_dev->lists_mutex);
1227 }
1228 
ib_uverbs_remove_one(struct ib_device * device,void * client_data)1229 static void ib_uverbs_remove_one(struct ib_device *device, void *client_data)
1230 {
1231 	struct ib_uverbs_device *uverbs_dev = client_data;
1232 	int wait_clients = 1;
1233 
1234 	if (!uverbs_dev)
1235 		return;
1236 
1237 	dev_set_drvdata(uverbs_dev->dev, NULL);
1238 	device_destroy(uverbs_class, uverbs_dev->cdev.dev);
1239 	cdev_del(&uverbs_dev->cdev);
1240 
1241 	if (uverbs_dev->devnum < IB_UVERBS_MAX_DEVICES)
1242 		clear_bit(uverbs_dev->devnum, dev_map);
1243 	else
1244 		clear_bit(uverbs_dev->devnum - IB_UVERBS_MAX_DEVICES, overflow_map);
1245 
1246 	if (device->disassociate_ucontext) {
1247 		/* We disassociate HW resources and immediately return.
1248 		 * Userspace will see a EIO errno for all future access.
1249 		 * Upon returning, ib_device may be freed internally and is not
1250 		 * valid any more.
1251 		 * uverbs_device is still available until all clients close
1252 		 * their files, then the uverbs device ref count will be zero
1253 		 * and its resources will be freed.
1254 		 * Note: At this point no more files can be opened since the
1255 		 * cdev was deleted, however active clients can still issue
1256 		 * commands and close their open files.
1257 		 */
1258 		rcu_assign_pointer(uverbs_dev->ib_dev, NULL);
1259 		ib_uverbs_free_hw_resources(uverbs_dev, device);
1260 		wait_clients = 0;
1261 	}
1262 
1263 	if (atomic_dec_and_test(&uverbs_dev->refcount))
1264 		ib_uverbs_comp_dev(uverbs_dev);
1265 	if (wait_clients)
1266 		wait_for_completion(&uverbs_dev->comp);
1267 	kobject_put(&uverbs_dev->kobj);
1268 }
1269 
uverbs_devnode(struct device * dev,umode_t * mode)1270 static char *uverbs_devnode(struct device *dev, umode_t *mode)
1271 {
1272 	if (mode)
1273 		*mode = 0666;
1274 	return kasprintf(GFP_KERNEL, "infiniband/%s", dev_name(dev));
1275 }
1276 
ib_uverbs_init(void)1277 static int __init ib_uverbs_init(void)
1278 {
1279 	int ret;
1280 
1281 	ret = register_chrdev_region(IB_UVERBS_BASE_DEV, IB_UVERBS_MAX_DEVICES,
1282 				     "infiniband_verbs");
1283 	if (ret) {
1284 		printk(KERN_ERR "user_verbs: couldn't register device number\n");
1285 		goto out;
1286 	}
1287 
1288 	uverbs_class = class_create(THIS_MODULE, "infiniband_verbs");
1289 	if (IS_ERR(uverbs_class)) {
1290 		ret = PTR_ERR(uverbs_class);
1291 		printk(KERN_ERR "user_verbs: couldn't create class infiniband_verbs\n");
1292 		goto out_chrdev;
1293 	}
1294 
1295 	uverbs_class->devnode = uverbs_devnode;
1296 
1297 	ret = class_create_file(uverbs_class, &class_attr_abi_version.attr);
1298 	if (ret) {
1299 		printk(KERN_ERR "user_verbs: couldn't create abi_version attribute\n");
1300 		goto out_class;
1301 	}
1302 
1303 	ret = ib_register_client(&uverbs_client);
1304 	if (ret) {
1305 		printk(KERN_ERR "user_verbs: couldn't register client\n");
1306 		goto out_class;
1307 	}
1308 
1309 	return 0;
1310 
1311 out_class:
1312 	class_destroy(uverbs_class);
1313 
1314 out_chrdev:
1315 	unregister_chrdev_region(IB_UVERBS_BASE_DEV, IB_UVERBS_MAX_DEVICES);
1316 
1317 out:
1318 	return ret;
1319 }
1320 
ib_uverbs_cleanup(void)1321 static void __exit ib_uverbs_cleanup(void)
1322 {
1323 	ib_unregister_client(&uverbs_client);
1324 	class_destroy(uverbs_class);
1325 	unregister_chrdev_region(IB_UVERBS_BASE_DEV, IB_UVERBS_MAX_DEVICES);
1326 	if (overflow_maj)
1327 		unregister_chrdev_region(overflow_maj, IB_UVERBS_MAX_DEVICES);
1328 	idr_destroy(&ib_uverbs_pd_idr);
1329 	idr_destroy(&ib_uverbs_mr_idr);
1330 	idr_destroy(&ib_uverbs_mw_idr);
1331 	idr_destroy(&ib_uverbs_ah_idr);
1332 	idr_destroy(&ib_uverbs_cq_idr);
1333 	idr_destroy(&ib_uverbs_qp_idr);
1334 	idr_destroy(&ib_uverbs_srq_idr);
1335 }
1336 
1337 module_init(ib_uverbs_init);
1338 module_exit(ib_uverbs_cleanup);
1339