Lines Matching refs:pmem
33 static struct device *to_dev(struct pmem_device *pmem) in to_dev() argument
39 return pmem->bb.dev; in to_dev()
42 static struct nd_region *to_region(struct pmem_device *pmem) in to_region() argument
44 return to_nd_region(to_dev(pmem)->parent); in to_region()
47 static void hwpoison_clear(struct pmem_device *pmem, in hwpoison_clear() argument
53 if (is_vmalloc_addr(pmem->virt_addr)) in hwpoison_clear()
71 static blk_status_t pmem_clear_poison(struct pmem_device *pmem, in pmem_clear_poison() argument
74 struct device *dev = to_dev(pmem); in pmem_clear_poison()
79 sector = (offset - pmem->data_offset) / 512; in pmem_clear_poison()
81 cleared = nvdimm_clear_poison(dev, pmem->phys_addr + offset, len); in pmem_clear_poison()
85 hwpoison_clear(pmem, pmem->phys_addr + offset, cleared); in pmem_clear_poison()
90 badblocks_clear(&pmem->bb, sector, cleared); in pmem_clear_poison()
91 if (pmem->bb_state) in pmem_clear_poison()
92 sysfs_notify_dirent(pmem->bb_state); in pmem_clear_poison()
95 arch_invalidate_pmem(pmem->virt_addr + offset, len); in pmem_clear_poison()
140 static blk_status_t pmem_do_read(struct pmem_device *pmem, in pmem_do_read() argument
145 phys_addr_t pmem_off = sector * 512 + pmem->data_offset; in pmem_do_read()
146 void *pmem_addr = pmem->virt_addr + pmem_off; in pmem_do_read()
148 if (unlikely(is_bad_pmem(&pmem->bb, sector, len))) in pmem_do_read()
156 static blk_status_t pmem_do_write(struct pmem_device *pmem, in pmem_do_write() argument
162 phys_addr_t pmem_off = sector * 512 + pmem->data_offset; in pmem_do_write()
163 void *pmem_addr = pmem->virt_addr + pmem_off; in pmem_do_write()
165 if (unlikely(is_bad_pmem(&pmem->bb, sector, len))) in pmem_do_write()
185 rc = pmem_clear_poison(pmem, pmem_off, len); in pmem_do_write()
200 struct pmem_device *pmem = bio->bi_disk->private_data; in pmem_submit_bio() local
201 struct nd_region *nd_region = to_region(pmem); in pmem_submit_bio()
211 rc = pmem_do_write(pmem, bvec.bv_page, bvec.bv_offset, in pmem_submit_bio()
214 rc = pmem_do_read(pmem, bvec.bv_page, bvec.bv_offset, in pmem_submit_bio()
237 struct pmem_device *pmem = bdev->bd_disk->private_data; in pmem_rw_page() local
241 rc = pmem_do_write(pmem, page, 0, sector, thp_size(page)); in pmem_rw_page()
243 rc = pmem_do_read(pmem, page, 0, sector, thp_size(page)); in pmem_rw_page()
257 __weak long __pmem_direct_access(struct pmem_device *pmem, pgoff_t pgoff, in __pmem_direct_access() argument
260 resource_size_t offset = PFN_PHYS(pgoff) + pmem->data_offset; in __pmem_direct_access()
262 if (unlikely(is_bad_pmem(&pmem->bb, PFN_PHYS(pgoff) / 512, in __pmem_direct_access()
267 *kaddr = pmem->virt_addr + offset; in __pmem_direct_access()
269 *pfn = phys_to_pfn_t(pmem->phys_addr + offset, pmem->pfn_flags); in __pmem_direct_access()
275 if (unlikely(pmem->bb.count)) in __pmem_direct_access()
277 return PHYS_PFN(pmem->size - pmem->pfn_pad - offset); in __pmem_direct_access()
289 struct pmem_device *pmem = dax_get_private(dax_dev); in pmem_dax_zero_page_range() local
291 return blk_status_to_errno(pmem_do_write(pmem, ZERO_PAGE(0), 0, in pmem_dax_zero_page_range()
299 struct pmem_device *pmem = dax_get_private(dax_dev); in pmem_dax_direct_access() local
301 return __pmem_direct_access(pmem, pgoff, nr_pages, kaddr, pfn); in pmem_dax_direct_access()
358 struct pmem_device *pmem = __pmem; in pmem_release_disk() local
360 kill_dax(pmem->dax_dev); in pmem_release_disk()
361 put_dax(pmem->dax_dev); in pmem_release_disk()
362 del_gendisk(pmem->disk); in pmem_release_disk()
363 put_disk(pmem->disk); in pmem_release_disk()
382 struct pmem_device *pmem; in pmem_attach_disk() local
390 pmem = devm_kzalloc(dev, sizeof(*pmem), GFP_KERNEL); in pmem_attach_disk()
391 if (!pmem) in pmem_attach_disk()
401 rc = nvdimm_setup_pfn(nd_pfn, &pmem->pgmap); in pmem_attach_disk()
409 dev_set_drvdata(dev, pmem); in pmem_attach_disk()
410 pmem->phys_addr = res->start; in pmem_attach_disk()
411 pmem->size = resource_size(res); in pmem_attach_disk()
428 pmem->pfn_flags = PFN_DEV; in pmem_attach_disk()
429 pmem->pgmap.ref = &q->q_usage_counter; in pmem_attach_disk()
431 pmem->pgmap.type = MEMORY_DEVICE_FS_DAX; in pmem_attach_disk()
432 pmem->pgmap.ops = &fsdax_pagemap_ops; in pmem_attach_disk()
433 addr = devm_memremap_pages(dev, &pmem->pgmap); in pmem_attach_disk()
435 pmem->data_offset = le64_to_cpu(pfn_sb->dataoff); in pmem_attach_disk()
436 pmem->pfn_pad = resource_size(res) - in pmem_attach_disk()
437 range_len(&pmem->pgmap.range); in pmem_attach_disk()
438 pmem->pfn_flags |= PFN_MAP; in pmem_attach_disk()
439 bb_range = pmem->pgmap.range; in pmem_attach_disk()
440 bb_range.start += pmem->data_offset; in pmem_attach_disk()
442 pmem->pgmap.range.start = res->start; in pmem_attach_disk()
443 pmem->pgmap.range.end = res->end; in pmem_attach_disk()
444 pmem->pgmap.nr_range = 1; in pmem_attach_disk()
445 pmem->pgmap.type = MEMORY_DEVICE_FS_DAX; in pmem_attach_disk()
446 pmem->pgmap.ops = &fsdax_pagemap_ops; in pmem_attach_disk()
447 addr = devm_memremap_pages(dev, &pmem->pgmap); in pmem_attach_disk()
448 pmem->pfn_flags |= PFN_MAP; in pmem_attach_disk()
449 bb_range = pmem->pgmap.range; in pmem_attach_disk()
451 addr = devm_memremap(dev, pmem->phys_addr, in pmem_attach_disk()
452 pmem->size, ARCH_MEMREMAP_PMEM); in pmem_attach_disk()
454 &pmem->pgmap)) in pmem_attach_disk()
462 pmem->virt_addr = addr; in pmem_attach_disk()
469 if (pmem->pfn_flags & PFN_MAP) in pmem_attach_disk()
475 pmem->disk = disk; in pmem_attach_disk()
480 disk->private_data = pmem; in pmem_attach_disk()
482 set_capacity(disk, (pmem->size - pmem->pfn_pad - pmem->data_offset) in pmem_attach_disk()
484 if (devm_init_badblocks(dev, &pmem->bb)) in pmem_attach_disk()
486 nvdimm_badblocks_populate(nd_region, &pmem->bb, &bb_range); in pmem_attach_disk()
487 disk->bb = &pmem->bb; in pmem_attach_disk()
491 dax_dev = alloc_dax(pmem, disk->disk_name, &pmem_dax_ops, flags); in pmem_attach_disk()
497 pmem->dax_dev = dax_dev; in pmem_attach_disk()
502 if (devm_add_action_or_reset(dev, pmem_release_disk, pmem)) in pmem_attach_disk()
507 pmem->bb_state = sysfs_get_dirent(disk_to_dev(disk)->kobj.sd, in pmem_attach_disk()
509 if (!pmem->bb_state) in pmem_attach_disk()
569 struct pmem_device *pmem = dev_get_drvdata(dev); in nd_pmem_remove() local
578 sysfs_put(pmem->bb_state); in nd_pmem_remove()
579 pmem->bb_state = NULL; in nd_pmem_remove()
613 struct pmem_device *pmem = dev_get_drvdata(dev); in nd_pmem_notify() local
615 nd_region = to_region(pmem); in nd_pmem_notify()
616 bb = &pmem->bb; in nd_pmem_notify()
617 bb_state = pmem->bb_state; in nd_pmem_notify()
624 offset = pmem->data_offset + in nd_pmem_notify()