1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */
3 #include <linux/init.h>
4 #include <linux/kernel.h>
5 #include <linux/module.h>
6 #include <linux/pci.h>
7 #include <linux/device.h>
8 #include <linux/sched/task.h>
9 #include <linux/intel-svm.h>
10 #include <linux/io-64-nonatomic-lo-hi.h>
11 #include <linux/cdev.h>
12 #include <linux/fs.h>
13 #include <linux/poll.h>
14 #include <uapi/linux/idxd.h>
15 #include "registers.h"
16 #include "idxd.h"
17
18 struct idxd_cdev_context {
19 const char *name;
20 dev_t devt;
21 struct ida minor_ida;
22 };
23
24 /*
25 * ictx is an array based off of accelerator types. enum idxd_type
26 * is used as index
27 */
28 static struct idxd_cdev_context ictx[IDXD_TYPE_MAX] = {
29 { .name = "dsa" },
30 };
31
32 struct idxd_user_context {
33 struct idxd_wq *wq;
34 struct task_struct *task;
35 unsigned int flags;
36 };
37
idxd_cdev_dev_release(struct device * dev)38 static void idxd_cdev_dev_release(struct device *dev)
39 {
40 struct idxd_cdev *idxd_cdev = container_of(dev, struct idxd_cdev, dev);
41 struct idxd_cdev_context *cdev_ctx;
42 struct idxd_wq *wq = idxd_cdev->wq;
43
44 cdev_ctx = &ictx[wq->idxd->type];
45 ida_simple_remove(&cdev_ctx->minor_ida, idxd_cdev->minor);
46 kfree(idxd_cdev);
47 }
48
49 static struct device_type idxd_cdev_device_type = {
50 .name = "idxd_cdev",
51 .release = idxd_cdev_dev_release,
52 };
53
inode_idxd_cdev(struct inode * inode)54 static inline struct idxd_cdev *inode_idxd_cdev(struct inode *inode)
55 {
56 struct cdev *cdev = inode->i_cdev;
57
58 return container_of(cdev, struct idxd_cdev, cdev);
59 }
60
inode_wq(struct inode * inode)61 static inline struct idxd_wq *inode_wq(struct inode *inode)
62 {
63 struct idxd_cdev *idxd_cdev = inode_idxd_cdev(inode);
64
65 return idxd_cdev->wq;
66 }
67
idxd_cdev_open(struct inode * inode,struct file * filp)68 static int idxd_cdev_open(struct inode *inode, struct file *filp)
69 {
70 struct idxd_user_context *ctx;
71 struct idxd_device *idxd;
72 struct idxd_wq *wq;
73 struct device *dev;
74 int rc = 0;
75
76 wq = inode_wq(inode);
77 idxd = wq->idxd;
78 dev = &idxd->pdev->dev;
79
80 dev_dbg(dev, "%s called: %d\n", __func__, idxd_wq_refcount(wq));
81
82 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
83 if (!ctx)
84 return -ENOMEM;
85
86 mutex_lock(&wq->wq_lock);
87
88 if (idxd_wq_refcount(wq) > 0 && wq_dedicated(wq)) {
89 rc = -EBUSY;
90 goto failed;
91 }
92
93 ctx->wq = wq;
94 filp->private_data = ctx;
95 idxd_wq_get(wq);
96 mutex_unlock(&wq->wq_lock);
97 return 0;
98
99 failed:
100 mutex_unlock(&wq->wq_lock);
101 kfree(ctx);
102 return rc;
103 }
104
idxd_cdev_release(struct inode * node,struct file * filep)105 static int idxd_cdev_release(struct inode *node, struct file *filep)
106 {
107 struct idxd_user_context *ctx = filep->private_data;
108 struct idxd_wq *wq = ctx->wq;
109 struct idxd_device *idxd = wq->idxd;
110 struct device *dev = &idxd->pdev->dev;
111
112 dev_dbg(dev, "%s called\n", __func__);
113 filep->private_data = NULL;
114
115 /* Wait for in-flight operations to complete. */
116 idxd_wq_drain(wq);
117
118 kfree(ctx);
119 mutex_lock(&wq->wq_lock);
120 idxd_wq_put(wq);
121 mutex_unlock(&wq->wq_lock);
122 return 0;
123 }
124
check_vma(struct idxd_wq * wq,struct vm_area_struct * vma,const char * func)125 static int check_vma(struct idxd_wq *wq, struct vm_area_struct *vma,
126 const char *func)
127 {
128 struct device *dev = &wq->idxd->pdev->dev;
129
130 if ((vma->vm_end - vma->vm_start) > PAGE_SIZE) {
131 dev_info_ratelimited(dev,
132 "%s: %s: mapping too large: %lu\n",
133 current->comm, func,
134 vma->vm_end - vma->vm_start);
135 return -EINVAL;
136 }
137
138 return 0;
139 }
140
idxd_cdev_mmap(struct file * filp,struct vm_area_struct * vma)141 static int idxd_cdev_mmap(struct file *filp, struct vm_area_struct *vma)
142 {
143 struct idxd_user_context *ctx = filp->private_data;
144 struct idxd_wq *wq = ctx->wq;
145 struct idxd_device *idxd = wq->idxd;
146 struct pci_dev *pdev = idxd->pdev;
147 phys_addr_t base = pci_resource_start(pdev, IDXD_WQ_BAR);
148 unsigned long pfn;
149 int rc;
150
151 dev_dbg(&pdev->dev, "%s called\n", __func__);
152 rc = check_vma(wq, vma, __func__);
153 if (rc < 0)
154 return rc;
155
156 vma->vm_flags |= VM_DONTCOPY;
157 pfn = (base + idxd_get_wq_portal_full_offset(wq->id,
158 IDXD_PORTAL_LIMITED)) >> PAGE_SHIFT;
159 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
160 vma->vm_private_data = ctx;
161
162 return io_remap_pfn_range(vma, vma->vm_start, pfn, PAGE_SIZE,
163 vma->vm_page_prot);
164 }
165
idxd_cdev_poll(struct file * filp,struct poll_table_struct * wait)166 static __poll_t idxd_cdev_poll(struct file *filp,
167 struct poll_table_struct *wait)
168 {
169 struct idxd_user_context *ctx = filp->private_data;
170 struct idxd_wq *wq = ctx->wq;
171 struct idxd_device *idxd = wq->idxd;
172 unsigned long flags;
173 __poll_t out = 0;
174
175 poll_wait(filp, &wq->err_queue, wait);
176 spin_lock_irqsave(&idxd->dev_lock, flags);
177 if (idxd->sw_err.valid)
178 out = EPOLLIN | EPOLLRDNORM;
179 spin_unlock_irqrestore(&idxd->dev_lock, flags);
180
181 return out;
182 }
183
184 static const struct file_operations idxd_cdev_fops = {
185 .owner = THIS_MODULE,
186 .open = idxd_cdev_open,
187 .release = idxd_cdev_release,
188 .mmap = idxd_cdev_mmap,
189 .poll = idxd_cdev_poll,
190 };
191
idxd_cdev_get_major(struct idxd_device * idxd)192 int idxd_cdev_get_major(struct idxd_device *idxd)
193 {
194 return MAJOR(ictx[idxd->type].devt);
195 }
196
idxd_wq_add_cdev(struct idxd_wq * wq)197 int idxd_wq_add_cdev(struct idxd_wq *wq)
198 {
199 struct idxd_device *idxd = wq->idxd;
200 struct idxd_cdev *idxd_cdev;
201 struct cdev *cdev;
202 struct device *dev;
203 struct idxd_cdev_context *cdev_ctx;
204 int rc, minor;
205
206 idxd_cdev = kzalloc(sizeof(*idxd_cdev), GFP_KERNEL);
207 if (!idxd_cdev)
208 return -ENOMEM;
209
210 idxd_cdev->wq = wq;
211 cdev = &idxd_cdev->cdev;
212 dev = &idxd_cdev->dev;
213 cdev_ctx = &ictx[wq->idxd->type];
214 minor = ida_simple_get(&cdev_ctx->minor_ida, 0, MINORMASK, GFP_KERNEL);
215 if (minor < 0) {
216 kfree(idxd_cdev);
217 return minor;
218 }
219 idxd_cdev->minor = minor;
220
221 device_initialize(dev);
222 dev->parent = &wq->conf_dev;
223 dev->bus = idxd_get_bus_type(idxd);
224 dev->type = &idxd_cdev_device_type;
225 dev->devt = MKDEV(MAJOR(cdev_ctx->devt), minor);
226
227 rc = dev_set_name(dev, "%s/wq%u.%u", idxd_get_dev_name(idxd),
228 idxd->id, wq->id);
229 if (rc < 0)
230 goto err;
231
232 wq->idxd_cdev = idxd_cdev;
233 cdev_init(cdev, &idxd_cdev_fops);
234 rc = cdev_device_add(cdev, dev);
235 if (rc) {
236 dev_dbg(&wq->idxd->pdev->dev, "cdev_add failed: %d\n", rc);
237 goto err;
238 }
239
240 return 0;
241
242 err:
243 put_device(dev);
244 wq->idxd_cdev = NULL;
245 return rc;
246 }
247
idxd_wq_del_cdev(struct idxd_wq * wq)248 void idxd_wq_del_cdev(struct idxd_wq *wq)
249 {
250 struct idxd_cdev *idxd_cdev;
251 struct idxd_cdev_context *cdev_ctx;
252
253 cdev_ctx = &ictx[wq->idxd->type];
254 idxd_cdev = wq->idxd_cdev;
255 wq->idxd_cdev = NULL;
256 cdev_device_del(&idxd_cdev->cdev, &idxd_cdev->dev);
257 put_device(&idxd_cdev->dev);
258 }
259
idxd_cdev_register(void)260 int idxd_cdev_register(void)
261 {
262 int rc, i;
263
264 for (i = 0; i < IDXD_TYPE_MAX; i++) {
265 ida_init(&ictx[i].minor_ida);
266 rc = alloc_chrdev_region(&ictx[i].devt, 0, MINORMASK,
267 ictx[i].name);
268 if (rc)
269 goto err_free_chrdev_region;
270 }
271
272 return 0;
273
274 err_free_chrdev_region:
275 for (i--; i >= 0; i--)
276 unregister_chrdev_region(ictx[i].devt, MINORMASK);
277
278 return rc;
279 }
280
idxd_cdev_remove(void)281 void idxd_cdev_remove(void)
282 {
283 int i;
284
285 for (i = 0; i < IDXD_TYPE_MAX; i++) {
286 unregister_chrdev_region(ictx[i].devt, MINORMASK);
287 ida_destroy(&ictx[i].minor_ida);
288 }
289 }
290