1 /*
2 * Copyright(c) 2016 - 2017 Intel Corporation. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of version 2 of the GNU General Public License as
6 * published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope that it will be useful, but
9 * WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11 * General Public License for more details.
12 */
13 #include <linux/pagemap.h>
14 #include <linux/module.h>
15 #include <linux/device.h>
16 #include <linux/pfn_t.h>
17 #include <linux/cdev.h>
18 #include <linux/slab.h>
19 #include <linux/dax.h>
20 #include <linux/fs.h>
21 #include <linux/mm.h>
22 #include "dax-private.h"
23 #include "dax.h"
24
25 static struct class *dax_class;
26
27 /*
28 * Rely on the fact that drvdata is set before the attributes are
29 * registered, and that the attributes are unregistered before drvdata
30 * is cleared to assume that drvdata is always valid.
31 */
id_show(struct device * dev,struct device_attribute * attr,char * buf)32 static ssize_t id_show(struct device *dev,
33 struct device_attribute *attr, char *buf)
34 {
35 struct dax_region *dax_region = dev_get_drvdata(dev);
36
37 return sprintf(buf, "%d\n", dax_region->id);
38 }
39 static DEVICE_ATTR_RO(id);
40
region_size_show(struct device * dev,struct device_attribute * attr,char * buf)41 static ssize_t region_size_show(struct device *dev,
42 struct device_attribute *attr, char *buf)
43 {
44 struct dax_region *dax_region = dev_get_drvdata(dev);
45
46 return sprintf(buf, "%llu\n", (unsigned long long)
47 resource_size(&dax_region->res));
48 }
49 static struct device_attribute dev_attr_region_size = __ATTR(size, 0444,
50 region_size_show, NULL);
51
align_show(struct device * dev,struct device_attribute * attr,char * buf)52 static ssize_t align_show(struct device *dev,
53 struct device_attribute *attr, char *buf)
54 {
55 struct dax_region *dax_region = dev_get_drvdata(dev);
56
57 return sprintf(buf, "%u\n", dax_region->align);
58 }
59 static DEVICE_ATTR_RO(align);
60
61 static struct attribute *dax_region_attributes[] = {
62 &dev_attr_region_size.attr,
63 &dev_attr_align.attr,
64 &dev_attr_id.attr,
65 NULL,
66 };
67
68 static const struct attribute_group dax_region_attribute_group = {
69 .name = "dax_region",
70 .attrs = dax_region_attributes,
71 };
72
73 static const struct attribute_group *dax_region_attribute_groups[] = {
74 &dax_region_attribute_group,
75 NULL,
76 };
77
dax_region_free(struct kref * kref)78 static void dax_region_free(struct kref *kref)
79 {
80 struct dax_region *dax_region;
81
82 dax_region = container_of(kref, struct dax_region, kref);
83 kfree(dax_region);
84 }
85
dax_region_put(struct dax_region * dax_region)86 void dax_region_put(struct dax_region *dax_region)
87 {
88 kref_put(&dax_region->kref, dax_region_free);
89 }
90 EXPORT_SYMBOL_GPL(dax_region_put);
91
dax_region_unregister(void * region)92 static void dax_region_unregister(void *region)
93 {
94 struct dax_region *dax_region = region;
95
96 sysfs_remove_groups(&dax_region->dev->kobj,
97 dax_region_attribute_groups);
98 dax_region_put(dax_region);
99 }
100
alloc_dax_region(struct device * parent,int region_id,struct resource * res,unsigned int align,void * addr,unsigned long pfn_flags)101 struct dax_region *alloc_dax_region(struct device *parent, int region_id,
102 struct resource *res, unsigned int align, void *addr,
103 unsigned long pfn_flags)
104 {
105 struct dax_region *dax_region;
106
107 /*
108 * The DAX core assumes that it can store its private data in
109 * parent->driver_data. This WARN is a reminder / safeguard for
110 * developers of device-dax drivers.
111 */
112 if (dev_get_drvdata(parent)) {
113 dev_WARN(parent, "dax core failed to setup private data\n");
114 return NULL;
115 }
116
117 if (!IS_ALIGNED(res->start, align)
118 || !IS_ALIGNED(resource_size(res), align))
119 return NULL;
120
121 dax_region = kzalloc(sizeof(*dax_region), GFP_KERNEL);
122 if (!dax_region)
123 return NULL;
124
125 dev_set_drvdata(parent, dax_region);
126 memcpy(&dax_region->res, res, sizeof(*res));
127 dax_region->pfn_flags = pfn_flags;
128 kref_init(&dax_region->kref);
129 dax_region->id = region_id;
130 ida_init(&dax_region->ida);
131 dax_region->align = align;
132 dax_region->dev = parent;
133 dax_region->base = addr;
134 if (sysfs_create_groups(&parent->kobj, dax_region_attribute_groups)) {
135 kfree(dax_region);
136 return NULL;;
137 }
138
139 kref_get(&dax_region->kref);
140 if (devm_add_action_or_reset(parent, dax_region_unregister, dax_region))
141 return NULL;
142 return dax_region;
143 }
144 EXPORT_SYMBOL_GPL(alloc_dax_region);
145
to_dev_dax(struct device * dev)146 static struct dev_dax *to_dev_dax(struct device *dev)
147 {
148 return container_of(dev, struct dev_dax, dev);
149 }
150
size_show(struct device * dev,struct device_attribute * attr,char * buf)151 static ssize_t size_show(struct device *dev,
152 struct device_attribute *attr, char *buf)
153 {
154 struct dev_dax *dev_dax = to_dev_dax(dev);
155 unsigned long long size = 0;
156 int i;
157
158 for (i = 0; i < dev_dax->num_resources; i++)
159 size += resource_size(&dev_dax->res[i]);
160
161 return sprintf(buf, "%llu\n", size);
162 }
163 static DEVICE_ATTR_RO(size);
164
165 static struct attribute *dev_dax_attributes[] = {
166 &dev_attr_size.attr,
167 NULL,
168 };
169
170 static const struct attribute_group dev_dax_attribute_group = {
171 .attrs = dev_dax_attributes,
172 };
173
174 static const struct attribute_group *dax_attribute_groups[] = {
175 &dev_dax_attribute_group,
176 NULL,
177 };
178
check_vma(struct dev_dax * dev_dax,struct vm_area_struct * vma,const char * func)179 static int check_vma(struct dev_dax *dev_dax, struct vm_area_struct *vma,
180 const char *func)
181 {
182 struct dax_region *dax_region = dev_dax->region;
183 struct device *dev = &dev_dax->dev;
184 unsigned long mask;
185
186 if (!dax_alive(dev_dax->dax_dev))
187 return -ENXIO;
188
189 /* prevent private mappings from being established */
190 if ((vma->vm_flags & VM_MAYSHARE) != VM_MAYSHARE) {
191 dev_info_ratelimited(dev,
192 "%s: %s: fail, attempted private mapping\n",
193 current->comm, func);
194 return -EINVAL;
195 }
196
197 mask = dax_region->align - 1;
198 if (vma->vm_start & mask || vma->vm_end & mask) {
199 dev_info_ratelimited(dev,
200 "%s: %s: fail, unaligned vma (%#lx - %#lx, %#lx)\n",
201 current->comm, func, vma->vm_start, vma->vm_end,
202 mask);
203 return -EINVAL;
204 }
205
206 if ((dax_region->pfn_flags & (PFN_DEV|PFN_MAP)) == PFN_DEV
207 && (vma->vm_flags & VM_DONTCOPY) == 0) {
208 dev_info_ratelimited(dev,
209 "%s: %s: fail, dax range requires MADV_DONTFORK\n",
210 current->comm, func);
211 return -EINVAL;
212 }
213
214 if (!vma_is_dax(vma)) {
215 dev_info_ratelimited(dev,
216 "%s: %s: fail, vma is not DAX capable\n",
217 current->comm, func);
218 return -EINVAL;
219 }
220
221 return 0;
222 }
223
224 /* see "strong" declaration in tools/testing/nvdimm/dax-dev.c */
dax_pgoff_to_phys(struct dev_dax * dev_dax,pgoff_t pgoff,unsigned long size)225 __weak phys_addr_t dax_pgoff_to_phys(struct dev_dax *dev_dax, pgoff_t pgoff,
226 unsigned long size)
227 {
228 struct resource *res;
229 /* gcc-4.6.3-nolibc for i386 complains that this is uninitialized */
230 phys_addr_t uninitialized_var(phys);
231 int i;
232
233 for (i = 0; i < dev_dax->num_resources; i++) {
234 res = &dev_dax->res[i];
235 phys = pgoff * PAGE_SIZE + res->start;
236 if (phys >= res->start && phys <= res->end)
237 break;
238 pgoff -= PHYS_PFN(resource_size(res));
239 }
240
241 if (i < dev_dax->num_resources) {
242 res = &dev_dax->res[i];
243 if (phys + size - 1 <= res->end)
244 return phys;
245 }
246
247 return -1;
248 }
249
__dev_dax_pte_fault(struct dev_dax * dev_dax,struct vm_fault * vmf)250 static int __dev_dax_pte_fault(struct dev_dax *dev_dax, struct vm_fault *vmf)
251 {
252 struct device *dev = &dev_dax->dev;
253 struct dax_region *dax_region;
254 int rc = VM_FAULT_SIGBUS;
255 phys_addr_t phys;
256 pfn_t pfn;
257 unsigned int fault_size = PAGE_SIZE;
258
259 if (check_vma(dev_dax, vmf->vma, __func__))
260 return VM_FAULT_SIGBUS;
261
262 dax_region = dev_dax->region;
263 if (dax_region->align > PAGE_SIZE) {
264 dev_dbg(dev, "%s: alignment (%#x) > fault size (%#x)\n",
265 __func__, dax_region->align, fault_size);
266 return VM_FAULT_SIGBUS;
267 }
268
269 if (fault_size != dax_region->align)
270 return VM_FAULT_SIGBUS;
271
272 phys = dax_pgoff_to_phys(dev_dax, vmf->pgoff, PAGE_SIZE);
273 if (phys == -1) {
274 dev_dbg(dev, "%s: pgoff_to_phys(%#lx) failed\n", __func__,
275 vmf->pgoff);
276 return VM_FAULT_SIGBUS;
277 }
278
279 pfn = phys_to_pfn_t(phys, dax_region->pfn_flags);
280
281 rc = vm_insert_mixed(vmf->vma, vmf->address, pfn);
282
283 if (rc == -ENOMEM)
284 return VM_FAULT_OOM;
285 if (rc < 0 && rc != -EBUSY)
286 return VM_FAULT_SIGBUS;
287
288 return VM_FAULT_NOPAGE;
289 }
290
__dev_dax_pmd_fault(struct dev_dax * dev_dax,struct vm_fault * vmf)291 static int __dev_dax_pmd_fault(struct dev_dax *dev_dax, struct vm_fault *vmf)
292 {
293 unsigned long pmd_addr = vmf->address & PMD_MASK;
294 struct device *dev = &dev_dax->dev;
295 struct dax_region *dax_region;
296 phys_addr_t phys;
297 pgoff_t pgoff;
298 pfn_t pfn;
299 unsigned int fault_size = PMD_SIZE;
300
301 if (check_vma(dev_dax, vmf->vma, __func__))
302 return VM_FAULT_SIGBUS;
303
304 dax_region = dev_dax->region;
305 if (dax_region->align > PMD_SIZE) {
306 dev_dbg(dev, "%s: alignment (%#x) > fault size (%#x)\n",
307 __func__, dax_region->align, fault_size);
308 return VM_FAULT_SIGBUS;
309 }
310
311 /* dax pmd mappings require pfn_t_devmap() */
312 if ((dax_region->pfn_flags & (PFN_DEV|PFN_MAP)) != (PFN_DEV|PFN_MAP)) {
313 dev_dbg(dev, "%s: region lacks devmap flags\n", __func__);
314 return VM_FAULT_SIGBUS;
315 }
316
317 if (fault_size < dax_region->align)
318 return VM_FAULT_SIGBUS;
319 else if (fault_size > dax_region->align)
320 return VM_FAULT_FALLBACK;
321
322 /* if we are outside of the VMA */
323 if (pmd_addr < vmf->vma->vm_start ||
324 (pmd_addr + PMD_SIZE) > vmf->vma->vm_end)
325 return VM_FAULT_SIGBUS;
326
327 pgoff = linear_page_index(vmf->vma, pmd_addr);
328 phys = dax_pgoff_to_phys(dev_dax, pgoff, PMD_SIZE);
329 if (phys == -1) {
330 dev_dbg(dev, "%s: pgoff_to_phys(%#lx) failed\n", __func__,
331 pgoff);
332 return VM_FAULT_SIGBUS;
333 }
334
335 pfn = phys_to_pfn_t(phys, dax_region->pfn_flags);
336
337 return vmf_insert_pfn_pmd(vmf->vma, vmf->address, vmf->pmd, pfn,
338 vmf->flags & FAULT_FLAG_WRITE);
339 }
340
341 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
__dev_dax_pud_fault(struct dev_dax * dev_dax,struct vm_fault * vmf)342 static int __dev_dax_pud_fault(struct dev_dax *dev_dax, struct vm_fault *vmf)
343 {
344 unsigned long pud_addr = vmf->address & PUD_MASK;
345 struct device *dev = &dev_dax->dev;
346 struct dax_region *dax_region;
347 phys_addr_t phys;
348 pgoff_t pgoff;
349 pfn_t pfn;
350 unsigned int fault_size = PUD_SIZE;
351
352
353 if (check_vma(dev_dax, vmf->vma, __func__))
354 return VM_FAULT_SIGBUS;
355
356 dax_region = dev_dax->region;
357 if (dax_region->align > PUD_SIZE) {
358 dev_dbg(dev, "%s: alignment (%#x) > fault size (%#x)\n",
359 __func__, dax_region->align, fault_size);
360 return VM_FAULT_SIGBUS;
361 }
362
363 /* dax pud mappings require pfn_t_devmap() */
364 if ((dax_region->pfn_flags & (PFN_DEV|PFN_MAP)) != (PFN_DEV|PFN_MAP)) {
365 dev_dbg(dev, "%s: region lacks devmap flags\n", __func__);
366 return VM_FAULT_SIGBUS;
367 }
368
369 if (fault_size < dax_region->align)
370 return VM_FAULT_SIGBUS;
371 else if (fault_size > dax_region->align)
372 return VM_FAULT_FALLBACK;
373
374 /* if we are outside of the VMA */
375 if (pud_addr < vmf->vma->vm_start ||
376 (pud_addr + PUD_SIZE) > vmf->vma->vm_end)
377 return VM_FAULT_SIGBUS;
378
379 pgoff = linear_page_index(vmf->vma, pud_addr);
380 phys = dax_pgoff_to_phys(dev_dax, pgoff, PUD_SIZE);
381 if (phys == -1) {
382 dev_dbg(dev, "%s: pgoff_to_phys(%#lx) failed\n", __func__,
383 pgoff);
384 return VM_FAULT_SIGBUS;
385 }
386
387 pfn = phys_to_pfn_t(phys, dax_region->pfn_flags);
388
389 return vmf_insert_pfn_pud(vmf->vma, vmf->address, vmf->pud, pfn,
390 vmf->flags & FAULT_FLAG_WRITE);
391 }
392 #else
__dev_dax_pud_fault(struct dev_dax * dev_dax,struct vm_fault * vmf)393 static int __dev_dax_pud_fault(struct dev_dax *dev_dax, struct vm_fault *vmf)
394 {
395 return VM_FAULT_FALLBACK;
396 }
397 #endif /* !CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */
398
dev_dax_huge_fault(struct vm_fault * vmf,enum page_entry_size pe_size)399 static int dev_dax_huge_fault(struct vm_fault *vmf,
400 enum page_entry_size pe_size)
401 {
402 int rc, id;
403 struct file *filp = vmf->vma->vm_file;
404 struct dev_dax *dev_dax = filp->private_data;
405
406 dev_dbg(&dev_dax->dev, "%s: %s: %s (%#lx - %#lx) size = %d\n", __func__,
407 current->comm, (vmf->flags & FAULT_FLAG_WRITE)
408 ? "write" : "read",
409 vmf->vma->vm_start, vmf->vma->vm_end, pe_size);
410
411 id = dax_read_lock();
412 switch (pe_size) {
413 case PE_SIZE_PTE:
414 rc = __dev_dax_pte_fault(dev_dax, vmf);
415 break;
416 case PE_SIZE_PMD:
417 rc = __dev_dax_pmd_fault(dev_dax, vmf);
418 break;
419 case PE_SIZE_PUD:
420 rc = __dev_dax_pud_fault(dev_dax, vmf);
421 break;
422 default:
423 rc = VM_FAULT_SIGBUS;
424 }
425 dax_read_unlock(id);
426
427 return rc;
428 }
429
dev_dax_fault(struct vm_fault * vmf)430 static int dev_dax_fault(struct vm_fault *vmf)
431 {
432 return dev_dax_huge_fault(vmf, PE_SIZE_PTE);
433 }
434
dev_dax_split(struct vm_area_struct * vma,unsigned long addr)435 static int dev_dax_split(struct vm_area_struct *vma, unsigned long addr)
436 {
437 struct file *filp = vma->vm_file;
438 struct dev_dax *dev_dax = filp->private_data;
439 struct dax_region *dax_region = dev_dax->region;
440
441 if (!IS_ALIGNED(addr, dax_region->align))
442 return -EINVAL;
443 return 0;
444 }
445
446 static const struct vm_operations_struct dax_vm_ops = {
447 .fault = dev_dax_fault,
448 .huge_fault = dev_dax_huge_fault,
449 .split = dev_dax_split,
450 };
451
dax_mmap(struct file * filp,struct vm_area_struct * vma)452 static int dax_mmap(struct file *filp, struct vm_area_struct *vma)
453 {
454 struct dev_dax *dev_dax = filp->private_data;
455 int rc, id;
456
457 dev_dbg(&dev_dax->dev, "%s\n", __func__);
458
459 /*
460 * We lock to check dax_dev liveness and will re-check at
461 * fault time.
462 */
463 id = dax_read_lock();
464 rc = check_vma(dev_dax, vma, __func__);
465 dax_read_unlock(id);
466 if (rc)
467 return rc;
468
469 vma->vm_ops = &dax_vm_ops;
470 vma->vm_flags |= VM_MIXEDMAP | VM_HUGEPAGE;
471 return 0;
472 }
473
474 /* return an unmapped area aligned to the dax region specified alignment */
dax_get_unmapped_area(struct file * filp,unsigned long addr,unsigned long len,unsigned long pgoff,unsigned long flags)475 static unsigned long dax_get_unmapped_area(struct file *filp,
476 unsigned long addr, unsigned long len, unsigned long pgoff,
477 unsigned long flags)
478 {
479 unsigned long off, off_end, off_align, len_align, addr_align, align;
480 struct dev_dax *dev_dax = filp ? filp->private_data : NULL;
481 struct dax_region *dax_region;
482
483 if (!dev_dax || addr)
484 goto out;
485
486 dax_region = dev_dax->region;
487 align = dax_region->align;
488 off = pgoff << PAGE_SHIFT;
489 off_end = off + len;
490 off_align = round_up(off, align);
491
492 if ((off_end <= off_align) || ((off_end - off_align) < align))
493 goto out;
494
495 len_align = len + align;
496 if ((off + len_align) < off)
497 goto out;
498
499 addr_align = current->mm->get_unmapped_area(filp, addr, len_align,
500 pgoff, flags);
501 if (!IS_ERR_VALUE(addr_align)) {
502 addr_align += (off - addr_align) & (align - 1);
503 return addr_align;
504 }
505 out:
506 return current->mm->get_unmapped_area(filp, addr, len, pgoff, flags);
507 }
508
dax_open(struct inode * inode,struct file * filp)509 static int dax_open(struct inode *inode, struct file *filp)
510 {
511 struct dax_device *dax_dev = inode_dax(inode);
512 struct inode *__dax_inode = dax_inode(dax_dev);
513 struct dev_dax *dev_dax = dax_get_private(dax_dev);
514
515 dev_dbg(&dev_dax->dev, "%s\n", __func__);
516 inode->i_mapping = __dax_inode->i_mapping;
517 inode->i_mapping->host = __dax_inode;
518 filp->f_mapping = inode->i_mapping;
519 filp->f_wb_err = filemap_sample_wb_err(filp->f_mapping);
520 filp->private_data = dev_dax;
521 inode->i_flags = S_DAX;
522
523 return 0;
524 }
525
dax_release(struct inode * inode,struct file * filp)526 static int dax_release(struct inode *inode, struct file *filp)
527 {
528 struct dev_dax *dev_dax = filp->private_data;
529
530 dev_dbg(&dev_dax->dev, "%s\n", __func__);
531 return 0;
532 }
533
534 static const struct file_operations dax_fops = {
535 .llseek = noop_llseek,
536 .owner = THIS_MODULE,
537 .open = dax_open,
538 .release = dax_release,
539 .get_unmapped_area = dax_get_unmapped_area,
540 .mmap = dax_mmap,
541 };
542
dev_dax_release(struct device * dev)543 static void dev_dax_release(struct device *dev)
544 {
545 struct dev_dax *dev_dax = to_dev_dax(dev);
546 struct dax_region *dax_region = dev_dax->region;
547 struct dax_device *dax_dev = dev_dax->dax_dev;
548
549 if (dev_dax->id >= 0)
550 ida_simple_remove(&dax_region->ida, dev_dax->id);
551 dax_region_put(dax_region);
552 put_dax(dax_dev);
553 kfree(dev_dax);
554 }
555
kill_dev_dax(struct dev_dax * dev_dax)556 static void kill_dev_dax(struct dev_dax *dev_dax)
557 {
558 struct dax_device *dax_dev = dev_dax->dax_dev;
559 struct inode *inode = dax_inode(dax_dev);
560
561 kill_dax(dax_dev);
562 unmap_mapping_range(inode->i_mapping, 0, 0, 1);
563 }
564
unregister_dev_dax(void * dev)565 static void unregister_dev_dax(void *dev)
566 {
567 struct dev_dax *dev_dax = to_dev_dax(dev);
568 struct dax_device *dax_dev = dev_dax->dax_dev;
569 struct inode *inode = dax_inode(dax_dev);
570 struct cdev *cdev = inode->i_cdev;
571
572 dev_dbg(dev, "%s\n", __func__);
573
574 kill_dev_dax(dev_dax);
575 cdev_device_del(cdev, dev);
576 put_device(dev);
577 }
578
devm_create_dev_dax(struct dax_region * dax_region,int id,struct resource * res,int count)579 struct dev_dax *devm_create_dev_dax(struct dax_region *dax_region,
580 int id, struct resource *res, int count)
581 {
582 struct device *parent = dax_region->dev;
583 struct dax_device *dax_dev;
584 struct dev_dax *dev_dax;
585 struct inode *inode;
586 struct device *dev;
587 struct cdev *cdev;
588 int rc, i;
589
590 if (!count)
591 return ERR_PTR(-EINVAL);
592
593 dev_dax = kzalloc(sizeof(*dev_dax) + sizeof(*res) * count, GFP_KERNEL);
594 if (!dev_dax)
595 return ERR_PTR(-ENOMEM);
596
597 for (i = 0; i < count; i++) {
598 if (!IS_ALIGNED(res[i].start, dax_region->align)
599 || !IS_ALIGNED(resource_size(&res[i]),
600 dax_region->align)) {
601 rc = -EINVAL;
602 break;
603 }
604 dev_dax->res[i].start = res[i].start;
605 dev_dax->res[i].end = res[i].end;
606 }
607
608 if (i < count)
609 goto err_id;
610
611 if (id < 0) {
612 id = ida_simple_get(&dax_region->ida, 0, 0, GFP_KERNEL);
613 dev_dax->id = id;
614 if (id < 0) {
615 rc = id;
616 goto err_id;
617 }
618 } else {
619 /* region provider owns @id lifetime */
620 dev_dax->id = -1;
621 }
622
623 /*
624 * No 'host' or dax_operations since there is no access to this
625 * device outside of mmap of the resulting character device.
626 */
627 dax_dev = alloc_dax(dev_dax, NULL, NULL);
628 if (!dax_dev) {
629 rc = -ENOMEM;
630 goto err_dax;
631 }
632
633 /* from here on we're committed to teardown via dax_dev_release() */
634 dev = &dev_dax->dev;
635 device_initialize(dev);
636
637 inode = dax_inode(dax_dev);
638 cdev = inode->i_cdev;
639 cdev_init(cdev, &dax_fops);
640 cdev->owner = parent->driver->owner;
641
642 dev_dax->num_resources = count;
643 dev_dax->dax_dev = dax_dev;
644 dev_dax->region = dax_region;
645 kref_get(&dax_region->kref);
646
647 dev->devt = inode->i_rdev;
648 dev->class = dax_class;
649 dev->parent = parent;
650 dev->groups = dax_attribute_groups;
651 dev->release = dev_dax_release;
652 dev_set_name(dev, "dax%d.%d", dax_region->id, id);
653
654 rc = cdev_device_add(cdev, dev);
655 if (rc) {
656 kill_dev_dax(dev_dax);
657 put_device(dev);
658 return ERR_PTR(rc);
659 }
660
661 rc = devm_add_action_or_reset(dax_region->dev, unregister_dev_dax, dev);
662 if (rc)
663 return ERR_PTR(rc);
664
665 return dev_dax;
666
667 err_dax:
668 if (dev_dax->id >= 0)
669 ida_simple_remove(&dax_region->ida, dev_dax->id);
670 err_id:
671 kfree(dev_dax);
672
673 return ERR_PTR(rc);
674 }
675 EXPORT_SYMBOL_GPL(devm_create_dev_dax);
676
dax_init(void)677 static int __init dax_init(void)
678 {
679 dax_class = class_create(THIS_MODULE, "dax");
680 return PTR_ERR_OR_ZERO(dax_class);
681 }
682
dax_exit(void)683 static void __exit dax_exit(void)
684 {
685 class_destroy(dax_class);
686 }
687
688 MODULE_AUTHOR("Intel Corporation");
689 MODULE_LICENSE("GPL v2");
690 subsys_initcall(dax_init);
691 module_exit(dax_exit);
692