71 static long dax_get_addr(struct buffer_head *bh, void __pmem **addr,  in dax_get_addr()  argument
75 	sector_t sector = bh->b_blocknr << (blkbits - 9);  in dax_get_addr()
76 	return bdev_direct_access(bh->b_bdev, sector, addr, &pfn, bh->b_size);  in dax_get_addr()
91 static bool buffer_written(struct buffer_head *bh)  in buffer_written()  argument
93 	return buffer_mapped(bh) && !buffer_unwritten(bh);  in buffer_written()
103 static bool buffer_size_valid(struct buffer_head *bh)  in buffer_size_valid()  argument
105 	return bh->b_state != 0;  in buffer_size_valid()
110 		      struct buffer_head *bh)  in dax_io()  argument
133 				bh->b_size = PAGE_ALIGN(end - pos);  in dax_io()
134 				bh->b_state = 0;  in dax_io()
135 				retval = get_block(inode, block, bh,  in dax_io()
139 				if (!buffer_size_valid(bh))  in dax_io()
140 					bh->b_size = 1 << blkbits;  in dax_io()
141 				bh_max = pos - first + bh->b_size;  in dax_io()
143 				unsigned done = bh->b_size -  in dax_io()
145 				bh->b_blocknr += done >> blkbits;  in dax_io()
146 				bh->b_size -= done;  in dax_io()
149 			hole = iov_iter_rw(iter) != WRITE && !buffer_written(bh);  in dax_io()
152 				size = bh->b_size - first;  in dax_io()
154 				retval = dax_get_addr(bh, &addr, blkbits);  in dax_io()
157 				if (buffer_unwritten(bh) || buffer_new(bh)) {  in dax_io()
213 	struct buffer_head bh;  in dax_do_io()  local
217 	memset(&bh, 0, sizeof(bh));  in dax_do_io()
233 	retval = dax_io(inode, iter, pos, end, get_block, &bh);  in dax_do_io()
239 		end_io(iocb, pos, retval, bh.b_private);  in dax_do_io()
278 static int copy_user_bh(struct page *to, struct buffer_head *bh,  in copy_user_bh()  argument
284 	if (dax_get_addr(bh, &vfrom, blkbits) < 0)  in copy_user_bh()
292 static int dax_insert_mapping(struct inode *inode, struct buffer_head *bh,  in dax_insert_mapping()  argument
296 	sector_t sector = bh->b_blocknr << (inode->i_blkbits - 9);  in dax_insert_mapping()
318 	error = bdev_direct_access(bh->b_bdev, sector, &addr, &pfn, bh->b_size);  in dax_insert_mapping()
326 	if (buffer_unwritten(bh) || buffer_new(bh)) {  in dax_insert_mapping()
362 	struct buffer_head bh;  in __dax_fault()  local
374 	memset(&bh, 0, sizeof(bh));  in __dax_fault()
376 	bh.b_size = PAGE_SIZE;  in __dax_fault()
401 	error = get_block(inode, block, &bh, 0);  in __dax_fault()
402 	if (!error && (bh.b_size < PAGE_SIZE))  in __dax_fault()
407 	if (!buffer_mapped(&bh) && !buffer_unwritten(&bh) && !vmf->cow_page) {  in __dax_fault()
409 			error = get_block(inode, block, &bh, 1);  in __dax_fault()
413 			if (!error && (bh.b_size < PAGE_SIZE))  in __dax_fault()
424 		if (buffer_written(&bh))  in __dax_fault()
425 			error = copy_user_bh(new_page, &bh, blkbits, vaddr);  in __dax_fault()
467 	error = dax_insert_mapping(inode, &bh, vma, vmf);  in __dax_fault()
468 	if (buffer_unwritten(&bh)) {  in __dax_fault()
470 			complete_unwritten(&bh, !error);  in __dax_fault()
533 	struct buffer_head bh;  in __dax_pmd_fault()  local
565 	memset(&bh, 0, sizeof(bh));  in __dax_pmd_fault()
568 	bh.b_size = PMD_SIZE;  in __dax_pmd_fault()
569 	length = get_block(inode, block, &bh, write);  in __dax_pmd_fault()
579 	if (!buffer_size_valid(&bh) || bh.b_size < PMD_SIZE)  in __dax_pmd_fault()
586 	if (buffer_new(&bh)) {  in __dax_pmd_fault()
606 	if (!write && !buffer_mapped(&bh) && buffer_uptodate(&bh)) {  in __dax_pmd_fault()
626 		sector = bh.b_blocknr << (blkbits - 9);  in __dax_pmd_fault()
627 		length = bdev_direct_access(bh.b_bdev, sector, &kaddr, &pfn,  in __dax_pmd_fault()
628 						bh.b_size);  in __dax_pmd_fault()
643 		if (buffer_unwritten(&bh) || buffer_new(&bh)) {  in __dax_pmd_fault()
659 	if (buffer_unwritten(&bh))  in __dax_pmd_fault()
660 		complete_unwritten(&bh, !(result & VM_FAULT_ERROR));  in __dax_pmd_fault()
739 	struct buffer_head bh;  in dax_zero_page_range()  local
749 	memset(&bh, 0, sizeof(bh));  in dax_zero_page_range()
750 	bh.b_size = PAGE_CACHE_SIZE;  in dax_zero_page_range()
751 	err = get_block(inode, index, &bh, 0);  in dax_zero_page_range()
754 	if (buffer_written(&bh)) {  in dax_zero_page_range()
756 		err = dax_get_addr(&bh, &addr, inode->i_blkbits);  in dax_zero_page_range()